2024-11-23 05:39:47,874 main DEBUG Apache Log4j Core 2.17.2 initializing configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 05:39:47,892 main DEBUG Took 0.015403 seconds to load 1 plugins from package org.apache.hadoop.hbase.logging 2024-11-23 05:39:47,893 main DEBUG PluginManager 'Core' found 129 plugins 2024-11-23 05:39:47,893 main DEBUG PluginManager 'Level' found 0 plugins 2024-11-23 05:39:47,895 main DEBUG PluginManager 'Lookup' found 16 plugins 2024-11-23 05:39:47,897 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,908 main DEBUG PluginManager 'TypeConverter' found 26 plugins 2024-11-23 05:39:47,929 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.util.MBeans", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,932 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,933 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.logging.TestJul2Slf4j", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,933 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,935 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.zookeeper", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,935 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,937 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSinkAdapter", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,937 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,938 main DEBUG LoggerConfig$Builder(additivity="null", level="ERROR", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsSystemImpl", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,939 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,940 main DEBUG LoggerConfig$Builder(additivity="false", level="WARN", levelAndRefs="null", name="org.apache.directory", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,941 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,942 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.ipc.FailedServers", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,942 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,943 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop.metrics2.impl.MetricsConfig", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,944 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,945 main DEBUG LoggerConfig$Builder(additivity="null", level="INFO", levelAndRefs="null", name="org.apache.hadoop.hbase.ScheduledChore", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,945 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,946 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase.regionserver.RSRpcServices", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,946 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,947 main DEBUG LoggerConfig$Builder(additivity="null", level="WARN", levelAndRefs="null", name="org.apache.hadoop", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,948 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,949 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hadoop.hbase", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,949 main DEBUG Building Plugin[name=logger, class=org.apache.logging.log4j.core.config.LoggerConfig]. 2024-11-23 05:39:47,950 main DEBUG LoggerConfig$Builder(additivity="null", level="DEBUG", levelAndRefs="null", name="org.apache.hbase.thirdparty.io.netty.channel", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,950 main DEBUG Building Plugin[name=root, class=org.apache.logging.log4j.core.config.LoggerConfig$RootLogger]. 2024-11-23 05:39:47,953 main DEBUG LoggerConfig$RootLogger$Builder(additivity="null", level="null", levelAndRefs="INFO,Console", includeLocation="null", ={}, ={}, Configuration(PropertiesConfig), Filter=null) 2024-11-23 05:39:47,955 main DEBUG Building Plugin[name=loggers, class=org.apache.logging.log4j.core.config.LoggersPlugin]. 2024-11-23 05:39:47,964 main DEBUG createLoggers(={org.apache.hadoop.metrics2.util.MBeans, org.apache.hadoop.hbase.logging.TestJul2Slf4j, org.apache.zookeeper, org.apache.hadoop.metrics2.impl.MetricsSinkAdapter, org.apache.hadoop.metrics2.impl.MetricsSystemImpl, org.apache.directory, org.apache.hadoop.hbase.ipc.FailedServers, org.apache.hadoop.metrics2.impl.MetricsConfig, org.apache.hadoop.hbase.ScheduledChore, org.apache.hadoop.hbase.regionserver.RSRpcServices, org.apache.hadoop, org.apache.hadoop.hbase, org.apache.hbase.thirdparty.io.netty.channel, root}) 2024-11-23 05:39:47,965 main DEBUG Building Plugin[name=layout, class=org.apache.logging.log4j.core.layout.PatternLayout]. 2024-11-23 05:39:47,967 main DEBUG PatternLayout$Builder(pattern="%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n", PatternSelector=null, Configuration(PropertiesConfig), Replace=null, charset="null", alwaysWriteExceptions="null", disableAnsi="null", noConsoleNoAnsi="null", header="null", footer="null") 2024-11-23 05:39:47,967 main DEBUG PluginManager 'Converter' found 47 plugins 2024-11-23 05:39:47,978 main DEBUG Building Plugin[name=appender, class=org.apache.hadoop.hbase.logging.HBaseTestAppender]. 2024-11-23 05:39:47,982 main DEBUG HBaseTestAppender$Builder(target="SYSTEM_ERR", maxSize="1G", bufferedIo="null", bufferSize="null", immediateFlush="null", ignoreExceptions="null", PatternLayout(%d{ISO8601} %-5p [%t%notEmpty{ %X}] %C{2}(%L): %m%n), name="Console", Configuration(PropertiesConfig), Filter=null, ={}) 2024-11-23 05:39:47,984 main DEBUG Starting HBaseTestOutputStreamManager SYSTEM_ERR 2024-11-23 05:39:47,985 main DEBUG Building Plugin[name=appenders, class=org.apache.logging.log4j.core.config.AppendersPlugin]. 2024-11-23 05:39:47,985 main DEBUG createAppenders(={Console}) 2024-11-23 05:39:47,986 main DEBUG Configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad initialized 2024-11-23 05:39:47,987 main DEBUG Starting configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad 2024-11-23 05:39:47,987 main DEBUG Started configuration org.apache.logging.log4j.core.config.properties.PropertiesConfiguration@47caedad OK. 2024-11-23 05:39:47,988 main DEBUG Shutting down OutputStreamManager SYSTEM_OUT.false.false-1 2024-11-23 05:39:47,988 main DEBUG OutputStream closed 2024-11-23 05:39:47,989 main DEBUG Shut down OutputStreamManager SYSTEM_OUT.false.false-1, all resources released: true 2024-11-23 05:39:47,989 main DEBUG Appender DefaultConsole-1 stopped with status true 2024-11-23 05:39:47,989 main DEBUG Stopped org.apache.logging.log4j.core.config.DefaultConfiguration@55634720 OK 2024-11-23 05:39:48,084 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6 2024-11-23 05:39:48,087 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=StatusLogger 2024-11-23 05:39:48,089 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=ContextSelector 2024-11-23 05:39:48,090 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name= 2024-11-23 05:39:48,091 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.directory 2024-11-23 05:39:48,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSinkAdapter 2024-11-23 05:39:48,092 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.zookeeper 2024-11-23 05:39:48,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.logging.TestJul2Slf4j 2024-11-23 05:39:48,093 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsSystemImpl 2024-11-23 05:39:48,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.util.MBeans 2024-11-23 05:39:48,094 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase 2024-11-23 05:39:48,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop 2024-11-23 05:39:48,095 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ipc.FailedServers 2024-11-23 05:39:48,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.regionserver.RSRpcServices 2024-11-23 05:39:48,096 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.metrics2.impl.MetricsConfig 2024-11-23 05:39:48,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hbase.thirdparty.io.netty.channel 2024-11-23 05:39:48,097 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Loggers,name=org.apache.hadoop.hbase.ScheduledChore 2024-11-23 05:39:48,098 main DEBUG Registering MBean org.apache.logging.log4j2:type=1dbd16a6,component=Appenders,name=Console 2024-11-23 05:39:48,102 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23 05:39:48,102 main DEBUG Reconfiguration complete for context[name=1dbd16a6] at URI jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-logging/target/hbase-logging-4.0.0-alpha-1-SNAPSHOT-tests.jar!/log4j2.properties (org.apache.logging.log4j.core.LoggerContext@19835e64) with optional ClassLoader: null 2024-11-23 05:39:48,103 main DEBUG Shutdown hook enabled. Registering a new one. 2024-11-23 05:39:48,104 main DEBUG LoggerContext[name=1dbd16a6, org.apache.logging.log4j.core.LoggerContext@19835e64] started OK. 2024-11-23T05:39:48,476 DEBUG [main {}] hbase.HBaseTestingUtil(323): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb 2024-11-23 05:39:48,480 main DEBUG AsyncLogger.ThreadNameStrategy=UNCACHED (user specified null, default is UNCACHED) 2024-11-23 05:39:48,480 main DEBUG org.apache.logging.log4j.core.util.SystemClock supports precise timestamps. 2024-11-23T05:39:48,490 INFO [main {}] hbase.HBaseClassTestRule(94): Test class org.apache.hadoop.hbase.TestAcidGuaranteesWithNoInMemCompaction timeout: 13 mins 2024-11-23T05:39:48,518 INFO [Time-limited test {}] hbase.HBaseTestingUtil(805): Starting up minicluster with option: StartMiniClusterOption{numMasters=1, masterClass=null, numRegionServers=1, rsPorts=, rsClass=null, numDataNodes=1, dataNodeHosts=null, numZkServers=1, createRootDir=false, createWALDir=false} 2024-11-23T05:39:48,537 INFO [Time-limited test {}] hbase.HBaseZKTestingUtil(84): Created new mini-cluster data directory: /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9, deleteOnExit=true 2024-11-23T05:39:48,538 INFO [Time-limited test {}] hbase.HBaseTestingUtil(818): STARTING DFS 2024-11-23T05:39:48,539 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting test.cache.data to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/test.cache.data in system properties and HBase conf 2024-11-23T05:39:48,540 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.tmp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/hadoop.tmp.dir in system properties and HBase conf 2024-11-23T05:39:48,541 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting hadoop.log.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/hadoop.log.dir in system properties and HBase conf 2024-11-23T05:39:48,542 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.local.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/mapreduce.cluster.local.dir in system properties and HBase conf 2024-11-23T05:39:48,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting mapreduce.cluster.temp.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/mapreduce.cluster.temp.dir in system properties and HBase conf 2024-11-23T05:39:48,543 INFO [Time-limited test {}] hbase.HBaseTestingUtil(738): read short circuit is OFF 2024-11-23T05:39:48,675 WARN [Time-limited test {}] util.NativeCodeLoader(60): Unable to load native-hadoop library for your platform... using builtin-java classes where applicable 2024-11-23T05:39:48,793 DEBUG [Time-limited test {}] fs.HFileSystem(310): The file system is not a DistributedFileSystem. Skipping on block location reordering 2024-11-23T05:39:48,797 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-labels.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/yarn.node-labels.fs-store.root-dir in system properties and HBase conf 2024-11-23T05:39:48,798 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.node-attribute.fs-store.root-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/yarn.node-attribute.fs-store.root-dir in system properties and HBase conf 2024-11-23T05:39:48,799 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.log-dirs to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/yarn.nodemanager.log-dirs in system properties and HBase conf 2024-11-23T05:39:48,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T05:39:48,800 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.active-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/yarn.timeline-service.entity-group-fs-store.active-dir in system properties and HBase conf 2024-11-23T05:39:48,801 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.timeline-service.entity-group-fs-store.done-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/yarn.timeline-service.entity-group-fs-store.done-dir in system properties and HBase conf 2024-11-23T05:39:48,802 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting yarn.nodemanager.remote-app-log-dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/yarn.nodemanager.remote-app-log-dir in system properties and HBase conf 2024-11-23T05:39:48,803 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T05:39:48,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.datanode.shared.file.descriptor.paths to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/dfs.datanode.shared.file.descriptor.paths in system properties and HBase conf 2024-11-23T05:39:48,804 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting nfs.dump.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/nfs.dump.dir in system properties and HBase conf 2024-11-23T05:39:48,805 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting java.io.tmpdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/java.io.tmpdir in system properties and HBase conf 2024-11-23T05:39:48,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.journalnode.edits.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/dfs.journalnode.edits.dir in system properties and HBase conf 2024-11-23T05:39:48,806 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting dfs.provided.aliasmap.inmemory.leveldb.dir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/dfs.provided.aliasmap.inmemory.leveldb.dir in system properties and HBase conf 2024-11-23T05:39:48,807 INFO [Time-limited test {}] hbase.HBaseTestingUtil(751): Setting fs.s3a.committer.staging.tmp.path to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/fs.s3a.committer.staging.tmp.path in system properties and HBase conf 2024-11-23T05:39:49,996 WARN [Time-limited test {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-namenode.properties,hadoop-metrics2.properties 2024-11-23T05:39:50,099 INFO [Time-limited test {}] log.Log(170): Logging initialized @3062ms to org.eclipse.jetty.util.log.Slf4jLog 2024-11-23T05:39:50,182 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T05:39:50,242 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T05:39:50,265 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T05:39:50,265 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T05:39:50,267 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T05:39:50,282 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T05:39:50,285 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/hadoop.log.dir/,AVAILABLE} 2024-11-23T05:39:50,286 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T05:39:50,555 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@1bbf242f{hdfs,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/java.io.tmpdir/jetty-localhost-36583-hadoop-hdfs-3_4_1-tests_jar-_-any-3575805734349694301/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T05:39:50,574 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:36583} 2024-11-23T05:39:50,575 INFO [Time-limited test {}] server.Server(415): Started @3540ms 2024-11-23T05:39:51,174 WARN [Time-limited test {}] server.AuthenticationFilter(240): Unable to initialize FileSignerSecretProvider, falling back to use random secrets. Reason: Could not read signature secret file: /home/jenkins/hadoop-http-auth-signature-secret 2024-11-23T05:39:51,184 INFO [Time-limited test {}] server.Server(375): jetty-9.4.53.v20231009; built: 2023-10-09T12:29:09.265Z; git: 27bde00a0b95a1d5bbee0eae7984f891d2d0f8c9; jvm 17.0.11+9 2024-11-23T05:39:51,186 INFO [Time-limited test {}] session.DefaultSessionIdManager(334): DefaultSessionIdManager workerName=node0 2024-11-23T05:39:51,186 INFO [Time-limited test {}] session.DefaultSessionIdManager(339): No SessionScavenger set, using defaults 2024-11-23T05:39:51,186 INFO [Time-limited test {}] session.HouseKeeper(132): node0 Scavenging every 660000ms 2024-11-23T05:39:51,187 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/hadoop.log.dir/,AVAILABLE} 2024-11-23T05:39:51,188 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,AVAILABLE} 2024-11-23T05:39:51,313 INFO [Time-limited test {}] handler.ContextHandler(921): Started o.e.j.w.WebAppContext@4d1d0948{datanode,/,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/java.io.tmpdir/jetty-localhost-34531-hadoop-hdfs-3_4_1-tests_jar-_-any-1843954399010361598/webapp/,AVAILABLE}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T05:39:51,314 INFO [Time-limited test {}] server.AbstractConnector(333): Started ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:34531} 2024-11-23T05:39:51,314 INFO [Time-limited test {}] server.Server(415): Started @4279ms 2024-11-23T05:39:51,377 WARN [Time-limited test {}] web.RestCsrfPreventionFilterHandler(75): Got null for restCsrfPreventionFilter - will not do any filtering. 2024-11-23T05:39:52,188 WARN [Thread-72 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/data/data1/current/BP-789294947-172.17.0.2-1732340389511/current, will proceed with Du for space computation calculation, 2024-11-23T05:39:52,188 WARN [Thread-73 {}] impl.BlockPoolSlice(347): dfsUsed file missing in /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/data/data2/current/BP-789294947-172.17.0.2-1732340389511/current, will proceed with Du for space computation calculation, 2024-11-23T05:39:52,219 WARN [Thread-58 {}] datanode.DirectoryScanner(302): dfs.datanode.directoryscan.throttle.limit.ms.per.sec set to value above 1000 ms/sec. Assuming default value of -1 2024-11-23T05:39:52,271 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7e63f962e2287b9 with lease ID 0xffa08b17e485fb9: Processing first storage report for DS-60affd37-2a73-4156-b54e-e57fb7a1828d from datanode DatanodeRegistration(127.0.0.1:36825, datanodeUuid=a2268abb-1a29-45ec-af1c-012ff91c8985, infoPort=35975, infoSecurePort=0, ipcPort=42033, storageInfo=lv=-57;cid=testClusterID;nsid=100985168;c=1732340389511) 2024-11-23T05:39:52,273 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7e63f962e2287b9 with lease ID 0xffa08b17e485fb9: from storage DS-60affd37-2a73-4156-b54e-e57fb7a1828d node DatanodeRegistration(127.0.0.1:36825, datanodeUuid=a2268abb-1a29-45ec-af1c-012ff91c8985, infoPort=35975, infoSecurePort=0, ipcPort=42033, storageInfo=lv=-57;cid=testClusterID;nsid=100985168;c=1732340389511), blocks: 0, hasStaleStorage: true, processing time: 2 msecs, invalidatedBlocks: 0 2024-11-23T05:39:52,274 INFO [Block report processor {}] blockmanagement.BlockManager(2940): BLOCK* processReport 0xf7e63f962e2287b9 with lease ID 0xffa08b17e485fb9: Processing first storage report for DS-8a1c4e54-4741-4e37-873b-6e62f6030b94 from datanode DatanodeRegistration(127.0.0.1:36825, datanodeUuid=a2268abb-1a29-45ec-af1c-012ff91c8985, infoPort=35975, infoSecurePort=0, ipcPort=42033, storageInfo=lv=-57;cid=testClusterID;nsid=100985168;c=1732340389511) 2024-11-23T05:39:52,274 INFO [Block report processor {}] blockmanagement.BlockManager(2972): BLOCK* processReport 0xf7e63f962e2287b9 with lease ID 0xffa08b17e485fb9: from storage DS-8a1c4e54-4741-4e37-873b-6e62f6030b94 node DatanodeRegistration(127.0.0.1:36825, datanodeUuid=a2268abb-1a29-45ec-af1c-012ff91c8985, infoPort=35975, infoSecurePort=0, ipcPort=42033, storageInfo=lv=-57;cid=testClusterID;nsid=100985168;c=1732340389511), blocks: 0, hasStaleStorage: false, processing time: 0 msecs, invalidatedBlocks: 0 2024-11-23T05:39:52,350 DEBUG [Time-limited test {}] hbase.HBaseTestingUtil(631): Setting hbase.rootdir to /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb 2024-11-23T05:39:52,428 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(261): Started connectionTimeout=30000, dir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/zookeeper_0, clientPort=53525, secureClientPort=-1, dataDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/zookeeper_0/version-2, dataDirSize=457 dataLogDir=/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/zookeeper_0/version-2, dataLogSize=457 tickTime=2000, maxClientCnxns=300, minSessionTimeout=4000, maxSessionTimeout=40000, clientPortListenBacklog=-1, serverId=0 2024-11-23T05:39:52,437 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(286): Started MiniZooKeeperCluster and ran 'stat' on client port=53525 2024-11-23T05:39:52,450 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T05:39:52,454 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T05:39:52,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741825_1001 (size=7) 2024-11-23T05:39:53,079 INFO [Time-limited test {}] util.FSUtils(489): Created version file at hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d with version=8 2024-11-23T05:39:53,080 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1139): Setting hbase.fs.tmp.dir to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/hbase-staging 2024-11-23T05:39:53,160 DEBUG [Time-limited test {}] channel.MultithreadEventLoopGroup(44): -Dio.netty.eventLoopThreads: 16 2024-11-23T05:39:53,413 INFO [Time-limited test {}] client.ConnectionUtils(128): master/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:39:53,423 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:39:53,424 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:39:53,430 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:39:53,430 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:39:53,430 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:39:53,621 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.MasterService, hbase.pb.RegionServerStatusService, hbase.pb.LockService, hbase.pb.HbckService, hbase.pb.ClientMetaService, hbase.pb.AdminService 2024-11-23T05:39:53,703 INFO [Time-limited test {}] metrics.MetricRegistriesLoader(60): Loaded MetricRegistries class org.apache.hadoop.hbase.metrics.impl.MetricRegistriesImpl 2024-11-23T05:39:53,716 DEBUG [Time-limited test {}] util.ClassSize(228): Using Unsafe to estimate memory layout 2024-11-23T05:39:53,721 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:39:53,752 DEBUG [Time-limited test {}] channel.DefaultChannelId(84): -Dio.netty.processId: 1533 (auto-detected) 2024-11-23T05:39:53,753 DEBUG [Time-limited test {}] channel.DefaultChannelId(106): -Dio.netty.machineId: 02:42:ac:ff:fe:11:00:02 (auto-detected) 2024-11-23T05:39:53,773 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:40189 2024-11-23T05:39:53,795 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=master:40189 connecting to ZooKeeper ensemble=127.0.0.1:53525 2024-11-23T05:39:53,882 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:401890x0, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:39:53,884 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): master:40189-0x101677e349d0000 connected 2024-11-23T05:39:53,980 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T05:39:53,982 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T05:39:53,992 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:39:53,997 INFO [Time-limited test {}] master.HMaster(525): hbase.rootdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d, hbase.cluster.distributed=false 2024-11-23T05:39:54,026 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:39:54,032 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=40189 2024-11-23T05:39:54,036 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=40189 2024-11-23T05:39:54,037 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=40189 2024-11-23T05:39:54,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=40189 2024-11-23T05:39:54,038 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=40189 2024-11-23T05:39:54,157 INFO [Time-limited test {}] client.ConnectionUtils(128): regionserver/9abfc7c2569e:0 server-side Connection retries=45 2024-11-23T05:39:54,159 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated default.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:39:54,159 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated priority.RWQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=2, maxQueueLength=30, handlerCount=3 2024-11-23T05:39:54,160 INFO [Time-limited test {}] ipc.RWQueueRpcExecutor(113): priority.RWQ.Fifo writeQueues=1 writeHandlers=1 readQueues=1 readHandlers=2 scanQueues=0 scanHandlers=0 2024-11-23T05:39:54,160 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated replication.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=3 2024-11-23T05:39:54,160 INFO [Time-limited test {}] ipc.RpcExecutor(188): Instantiated metaPriority.FPBQ.Fifo with queueClass=class java.util.concurrent.LinkedBlockingQueue; numCallQueues=1, maxQueueLength=30, handlerCount=1 2024-11-23T05:39:54,164 INFO [Time-limited test {}] ipc.RpcServerFactory(64): Creating org.apache.hadoop.hbase.ipc.NettyRpcServer hosting hbase.pb.ClientService, hbase.pb.AdminService, hbase.pb.ClientMetaService, hbase.pb.BootstrapNodeService 2024-11-23T05:39:54,166 INFO [Time-limited test {}] ipc.NettyRpcServer(309): Using org.apache.hbase.thirdparty.io.netty.buffer.PooledByteBufAllocator for buffer allocation 2024-11-23T05:39:54,173 INFO [Time-limited test {}] ipc.NettyRpcServer(191): Bind to /172.17.0.2:45719 2024-11-23T05:39:54,176 INFO [Time-limited test {}] zookeeper.RecoverableZooKeeper(137): Process identifier=regionserver:45719 connecting to ZooKeeper ensemble=127.0.0.1:53525 2024-11-23T05:39:54,178 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T05:39:54,182 INFO [Time-limited test {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T05:39:54,219 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:457190x0, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=None, state=SyncConnected, path=null 2024-11-23T05:39:54,220 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:457190x0, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:39:54,220 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKWatcher(635): regionserver:45719-0x101677e349d0001 connected 2024-11-23T05:39:54,224 INFO [Time-limited test {}] hfile.BlockCacheFactory(123): Allocating BlockCache size=880 MB, blockSize=64 KB 2024-11-23T05:39:54,236 DEBUG [Time-limited test {}] mob.MobFileCache(124): MobFileCache enabled with cacheSize=1000, evictPeriods=3600sec, evictRemainRatio=0.5 2024-11-23T05:39:54,240 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/master 2024-11-23T05:39:54,245 DEBUG [Time-limited test {}] zookeeper.ZKUtil(113): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/acl 2024-11-23T05:39:54,246 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=default.FPBQ.Fifo, numCallQueues=1, port=45719 2024-11-23T05:39:54,250 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=priority.RWQ.Fifo.write, numCallQueues=1, port=45719 2024-11-23T05:39:54,251 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=priority.RWQ.Fifo.read, numCallQueues=1, port=45719 2024-11-23T05:39:54,257 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=3 with threadPrefix=replication.FPBQ.Fifo, numCallQueues=1, port=45719 2024-11-23T05:39:54,258 DEBUG [Time-limited test {}] ipc.RpcExecutor(290): Started handlerCount=1 with threadPrefix=metaPriority.FPBQ.Fifo, numCallQueues=1, port=45719 2024-11-23T05:39:54,276 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:M:0;9abfc7c2569e:40189 2024-11-23T05:39:54,277 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(2510): Adding backup master ZNode /hbase/backup-masters/9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:54,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:39:54,291 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:39:54,292 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on existing znode=/hbase/backup-masters/9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:54,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:54,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/master 2024-11-23T05:39:54,319 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:54,320 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(111): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on existing znode=/hbase/master 2024-11-23T05:39:54,322 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ActiveMasterManager(245): Deleting ZNode for /hbase/backup-masters/9abfc7c2569e,40189,1732340393249 from backup master directory 2024-11-23T05:39:54,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:39:54,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/backup-masters/9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:54,332 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/backup-masters 2024-11-23T05:39:54,332 WARN [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:39:54,333 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ActiveMasterManager(255): Registered as active master=9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:54,336 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating data MemStoreChunkPool with chunk size 2 MB, max count 396, initial count 0 2024-11-23T05:39:54,338 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.ChunkCreator(488): Allocating index MemStoreChunkPool with chunk size 204.80 KB, max count 440, initial count 0 2024-11-23T05:39:54,392 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(620): Create cluster ID file [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/hbase.id] with ID: 1659c1c0-0bd8-44bc-ada4-34dd97dbfe67 2024-11-23T05:39:54,392 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(625): Write the cluster ID file to a temporary location: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/.tmp/hbase.id 2024-11-23T05:39:54,402 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741826_1002 (size=42) 2024-11-23T05:39:54,807 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSUtils(634): Move the temporary cluster ID file to its target location [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/.tmp/hbase.id]:[hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/hbase.id] 2024-11-23T05:39:54,859 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] fs.HFileSystem(339): Added intercepting call to namenode#getBlockLocations so can do block reordering using class org.apache.hadoop.hbase.fs.HFileSystem$ReorderWALBlocks 2024-11-23T05:39:54,865 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSTableDescriptors(270): Fetching table descriptors from the filesystem. 2024-11-23T05:39:54,883 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] util.FSTableDescriptors(299): Fetched table descriptors(size=0) cost 16ms. 2024-11-23T05:39:54,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:54,936 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:54,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741827_1003 (size=196) 2024-11-23T05:39:55,378 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(370): Create or load local region for table 'master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:39:55,380 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(132): Injected flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000 2024-11-23T05:39:55,393 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(244): No decryptEncryptedDataEncryptionKey method in DFSClient, should be hadoop version with HDFS-12396 java.lang.NoSuchMethodException: org.apache.hadoop.hdfs.DFSClient.decryptEncryptedDataEncryptionKey(org.apache.hadoop.fs.FileEncryptionInfo) at java.lang.Class.getDeclaredMethod(Class.java:2675) ~[?:?] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelperWithoutHDFS12396(FanOutOneBlockAsyncDFSOutputSaslHelper.java:183) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.createTransparentCryptoHelper(FanOutOneBlockAsyncDFSOutputSaslHelper.java:242) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.io.asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper.(FanOutOneBlockAsyncDFSOutputSaslHelper.java:253) ~[hbase-asyncfs-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Class.forName0(Native Method) ~[?:?] at java.lang.Class.forName(Class.java:375) ~[?:?] at org.apache.hadoop.hbase.wal.AsyncFSWALProvider.load(AsyncFSWALProvider.java:150) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.getProviderClass(WALFactory.java:174) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:262) ~[classes/:?] at org.apache.hadoop.hbase.wal.WALFactory.(WALFactory.java:231) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegion.create(MasterRegion.java:400) ~[classes/:?] at org.apache.hadoop.hbase.master.region.MasterRegionFactory.create(MasterRegionFactory.java:135) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.finishActiveMasterInitialization(HMaster.java:1003) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.startActiveMasterManager(HMaster.java:2535) ~[classes/:?] at org.apache.hadoop.hbase.master.HMaster.lambda$run$0(HMaster.java:613) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.lambda$tracedRunnable$2(TraceUtil.java:155) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:39:55,396 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:39:55,423 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741828_1004 (size=1189) 2024-11-23T05:39:55,845 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7590): Creating {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='master:store', {TABLE_ATTRIBUTES => {METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, under table dir hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store 2024-11-23T05:39:55,862 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741829_1005 (size=34) 2024-11-23T05:39:56,276 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] throttle.StoreHotnessProtector(112): StoreHotnessProtector is disabled. Set hbase.region.store.parallel.put.limit > 0 to enable, which may help mitigate load under heavy write pressure. 2024-11-23T05:39:56,279 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:39:56,280 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T05:39:56,280 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:39:56,280 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:39:56,282 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T05:39:56,282 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:39:56,282 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:39:56,283 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732340396280Disabling compacts and flushes for region at 1732340396280Disabling writes for close at 1732340396282 (+2 ms)Writing region close event to WAL at 1732340396282Closed at 1732340396282 2024-11-23T05:39:56,286 WARN [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(249): failed to clean up initializing flag: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/.initializing 2024-11-23T05:39:56,286 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegion(219): WALDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/WALs/9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:56,295 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:39:56,312 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9abfc7c2569e%2C40189%2C1732340393249, suffix=, logDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/WALs/9abfc7c2569e,40189,1732340393249, archiveDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/oldWALs, maxLogs=10 2024-11-23T05:39:56,348 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/WALs/9abfc7c2569e,40189,1732340393249/9abfc7c2569e%2C40189%2C1732340393249.1732340396319, exclude list is [], retry=0 2024-11-23T05:39:56,368 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36825,DS-60affd37-2a73-4156-b54e-e57fb7a1828d,DISK] 2024-11-23T05:39:56,371 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.ProtobufDecoder(117): Hadoop 3.3 and above shades protobuf. 2024-11-23T05:39:56,411 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/WALs/9abfc7c2569e,40189,1732340393249/9abfc7c2569e%2C40189%2C1732340393249.1732340396319 2024-11-23T05:39:56,412 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35975:35975)] 2024-11-23T05:39:56,413 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7752): Opening region: {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:39:56,413 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(898): Instantiated master:store,,1.1595e783b53d99cd5eef43b6debb2682.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:39:56,416 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7794): checking encryption for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,418 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(7797): checking classloading for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,459 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,483 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName info 2024-11-23T05:39:56,488 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:56,491 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:56,492 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family proc of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,495 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName proc 2024-11-23T05:39:56,496 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:56,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/proc, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:39:56,497 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rs of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,501 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName rs 2024-11-23T05:39:56,502 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:56,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/rs, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:39:56,503 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family state of region 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,507 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1595e783b53d99cd5eef43b6debb2682 columnFamilyName state 2024-11-23T05:39:56,507 DEBUG [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:56,508 INFO [StoreOpener-1595e783b53d99cd5eef43b6debb2682-1 {}] regionserver.HStore(327): Store=1595e783b53d99cd5eef43b6debb2682/state, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:39:56,509 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1038): replaying wal for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,513 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,514 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,520 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1048): stopping wal replay for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,521 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1060): Cleaning up temporary data for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,526 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table master:store descriptor;using region.getMemStoreFlushHeapSize/# of families (32.0 M)) instead. 2024-11-23T05:39:56,531 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1093): writing seq id for 1595e783b53d99cd5eef43b6debb2682 2024-11-23T05:39:56,538 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:39:56,540 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1114): Opened 1595e783b53d99cd5eef43b6debb2682; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=70609147, jitterRate=0.052158281207084656}, FlushLargeStoresPolicy{flushSizeLowerBound=33554432} 2024-11-23T05:39:56,548 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] regionserver.HRegion(1006): Region open journal for 1595e783b53d99cd5eef43b6debb2682: Writing region info on filesystem at 1732340396432Initializing all the Stores at 1732340396434 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732340396435 (+1 ms)Instantiating store for column family {NAME => 'proc', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340396435Instantiating store for column family {NAME => 'rs', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340396436 (+1 ms)Instantiating store for column family {NAME => 'state', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340396436Cleaning up temporary data from old regions at 1732340396521 (+85 ms)Region opened successfully at 1732340396548 (+27 ms) 2024-11-23T05:39:56,550 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.MasterRegionFlusherAndCompactor(122): Constructor flushSize=134217728, flushPerChanges=1000000, flushIntervalMs=900000, compactMin=4 2024-11-23T05:39:56,589 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d5bf8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:39:56,627 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(912): No meta location available on zookeeper, skip migrating... 2024-11-23T05:39:56,641 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.RegionProcedureStore(104): Starting the Region Procedure Store, number threads=5 2024-11-23T05:39:56,641 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(626): Starting 5 core workers (bigger of cpus/4 or 16) with max (burst) worker count=50 2024-11-23T05:39:56,646 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] region.RegionProcedureStore(255): Starting Region Procedure Store lease recovery... 2024-11-23T05:39:56,648 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(676): Recovered RegionProcedureStore lease in 2 msec 2024-11-23T05:39:56,653 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(690): Loaded RegionProcedureStore in 4 msec 2024-11-23T05:39:56,653 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.RemoteProcedureDispatcher(96): Instantiated, coreThreads=3 (allowCoreThreadTimeOut=true), queueMaxSize=32, operationDelay=150 2024-11-23T05:39:56,684 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.RegionServerTracker(127): Upgrading RegionServerTracker to active master mode; 0 have existingServerCrashProcedures, 0 possibly 'live' servers, and 0 'splitting'. 2024-11-23T05:39:56,696 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Unable to get data of znode /hbase/balancer because node does not exist (not necessarily an error) 2024-11-23T05:39:56,744 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/balancer already deleted, retry=false 2024-11-23T05:39:56,747 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] normalizer.SimpleRegionNormalizer(163): Updated configuration for key 'hbase.normalizer.merge.min_region_size.mb' from 0 to 1 2024-11-23T05:39:56,749 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Unable to get data of znode /hbase/normalizer because node does not exist (not necessarily an error) 2024-11-23T05:39:56,760 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/normalizer already deleted, retry=false 2024-11-23T05:39:56,762 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] normalizer.RegionNormalizerWorker(137): Normalizer rate limit set to unlimited 2024-11-23T05:39:56,767 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Unable to get data of znode /hbase/switch/split because node does not exist (not necessarily an error) 2024-11-23T05:39:56,777 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/split already deleted, retry=false 2024-11-23T05:39:56,778 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Unable to get data of znode /hbase/switch/merge because node does not exist (not necessarily an error) 2024-11-23T05:39:56,789 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/switch/merge already deleted, retry=false 2024-11-23T05:39:56,807 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKUtil(444): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Unable to get data of znode /hbase/snapshot-cleanup because node does not exist (not necessarily an error) 2024-11-23T05:39:56,818 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/snapshot-cleanup already deleted, retry=false 2024-11-23T05:39:56,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:39:56,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeCreated, state=SyncConnected, path=/hbase/running 2024-11-23T05:39:56,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:56,831 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:56,835 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(856): Active/primary master=9abfc7c2569e,40189,1732340393249, sessionid=0x101677e349d0000, setting cluster-up flag (Was=false) 2024-11-23T05:39:56,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:56,864 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:56,898 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/flush-table-proc/acquired, /hbase/flush-table-proc/reached, /hbase/flush-table-proc/abort 2024-11-23T05:39:56,901 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:56,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:56,923 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:56,948 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureUtil(251): Clearing all znodes /hbase/online-snapshot/acquired, /hbase/online-snapshot/reached, /hbase/online-snapshot/abort 2024-11-23T05:39:56,950 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure.ZKProcedureCoordinator(245): Starting controller for procedure member=9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:56,955 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ServerManager(1185): No .lastflushedseqids found at hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/.lastflushedseqids will record last flushed sequence id for regions by regionserver report all over again 2024-11-23T05:39:56,967 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(746): ClusterId : 1659c1c0-0bd8-44bc-ada4-34dd97dbfe67 2024-11-23T05:39:56,969 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure flush-table-proc initializing 2024-11-23T05:39:56,983 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure flush-table-proc initialized 2024-11-23T05:39:56,983 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(43): Procedure online-snapshot initializing 2024-11-23T05:39:56,995 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(45): Procedure online-snapshot initialized 2024-11-23T05:39:56,995 DEBUG [RS:0;9abfc7c2569e:45719 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@524399ba, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=9abfc7c2569e/172.17.0.2:0 2024-11-23T05:39:57,012 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.ShutdownHook(81): Installed shutdown hook thread: Shutdownhook:RS:0;9abfc7c2569e:45719 2024-11-23T05:39:57,014 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.RegionServerCoprocessorHost(66): System coprocessor loading is enabled 2024-11-23T05:39:57,014 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.RegionServerCoprocessorHost(67): Table coprocessor loading is enabled 2024-11-23T05:39:57,015 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(832): About to register with Master. 2024-11-23T05:39:57,017 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(2659): reportForDuty to master=9abfc7c2569e,40189,1732340393249 with port=45719, startcode=1732340394120 2024-11-23T05:39:57,029 DEBUG [RS:0;9abfc7c2569e:45719 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=RegionServerStatusService, sasl=false 2024-11-23T05:39:57,032 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.ProcedureExecutor(1139): Stored pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=false; InitMetaProcedure table=hbase:meta 2024-11-23T05:39:57,040 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.BaseLoadBalancer(416): slop=0.2 2024-11-23T05:39:57,047 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.StochasticLoadBalancer(272): Loaded config; maxSteps=1000000, runMaxSteps=false, stepsPerRegion=800, maxRunningTime=30000, isByTable=false, CostFunctions=[RegionCountSkewCostFunction, PrimaryRegionCountSkewCostFunction, MoveCostFunction, ServerLocalityCostFunction, RackLocalityCostFunction, TableSkewCostFunction, RegionReplicaHostCostFunction, RegionReplicaRackCostFunction, ReadRequestCostFunction, CPRequestCostFunction, WriteRequestCostFunction, MemStoreSizeCostFunction, StoreFileCostFunction] , sum of multiplier of cost functions = 0.0 etc. 2024-11-23T05:39:57,053 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] balancer.RegionHDFSBlockLocationFinder(133): Skipping locality-based refresh due to oldStatus=null, newStatus=Master: 9abfc7c2569e,40189,1732340393249 Number of backup masters: 0 Number of live region servers: 0 Number of dead region servers: 0 Number of unknown region servers: 0 Average load: 0.0 Number of requests: 0 Number of regions: 0 Number of regions in transition: 0 2024-11-23T05:39:57,060 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:56643, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins.hfs.0 (auth:SIMPLE), service=RegionServerStatusService 2024-11-23T05:39:57,061 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_OPEN_REGION-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:39:57,061 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_CLOSE_REGION-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:39:57,061 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SERVER_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:39:57,061 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_META_SERVER_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=5, maxPoolSize=5 2024-11-23T05:39:57,061 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=M_LOG_REPLAY_OPS-master/9abfc7c2569e:0, corePoolSize=10, maxPoolSize=10 2024-11-23T05:39:57,062 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_SNAPSHOT_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,062 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_MERGE_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:39:57,062 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] executor.ExecutorService(95): Starting executor service name=MASTER_TABLE_OPERATIONS-master/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,064 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.procedure2.CompletedProcedureCleaner; timeout=30000, timestamp=1732340427064 2024-11-23T05:39:57,065 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): log_cleaner Cleaner pool size is 1 2024-11-23T05:39:57,065 DEBUG [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40189 {}] ipc.MetricsHBaseServer(152): Unknown exception type org.apache.hadoop.hbase.ipc.ServerNotRunningYetException: Server is not running yet at org.apache.hadoop.hbase.master.HMaster.checkServiceStarted(HMaster.java:3334) ~[classes/:?] at org.apache.hadoop.hbase.master.MasterRpcServices.regionServerStartup(MasterRpcServices.java:667) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16714) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:39:57,067 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveLogCleaner 2024-11-23T05:39:57,068 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_WRITE_FS_LAYOUT, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:39:57,068 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(76): BOOTSTRAP: creating hbase:meta region 2024-11-23T05:39:57,071 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.replication.master.ReplicationLogCleaner 2024-11-23T05:39:57,071 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreWALCleaner 2024-11-23T05:39:57,071 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveProcedureWALCleaner 2024-11-23T05:39:57,071 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.LogCleaner(148): Creating 1 old WALs cleaner threads 2024-11-23T05:39:57,075 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:57,074 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=LogsCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,075 INFO [PEWorker-1 {}] util.FSTableDescriptors(156): Creating new hbase:meta table descriptor 'hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} 2024-11-23T05:39:57,077 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.DirScanPool(74): hfile_cleaner Cleaner pool size is 2 2024-11-23T05:39:57,078 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveMasterLocalStoreHFileCleaner 2024-11-23T05:39:57,079 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.HFileLinkCleaner 2024-11-23T05:39:57,080 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.snapshot.SnapshotHFileCleaner 2024-11-23T05:39:57,081 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.CleanerChore(192): Initialize cleaner=org.apache.hadoop.hbase.master.cleaner.TimeToLiveHFileCleaner 2024-11-23T05:39:57,082 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.HFileCleaner(260): Starting for large file=Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732340397082,5,FailOnTimeoutGroup] 2024-11-23T05:39:57,083 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] cleaner.HFileCleaner(275): Starting for small files=Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732340397083,5,FailOnTimeoutGroup] 2024-11-23T05:39:57,083 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HFileCleaner, period=600000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,083 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1741): Reopening regions with very high storeFileRefCount is disabled. Provide threshold value > 0 for hbase.regions.recovery.store.file.ref.count to enable it. 2024-11-23T05:39:57,085 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationBarrierCleaner, period=43200000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,085 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741831_1007 (size=1321) 2024-11-23T05:39:57,085 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=SnapshotCleaner, period=1800000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,092 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(2683): Master is not running yet 2024-11-23T05:39:57,092 WARN [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(841): reportForDuty failed; sleeping 100 ms and then retrying. 2024-11-23T05:39:57,195 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(2659): reportForDuty to master=9abfc7c2569e,40189,1732340393249 with port=45719, startcode=1732340394120 2024-11-23T05:39:57,199 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40189 {}] master.ServerManager(363): Checking decommissioned status of RegionServer 9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:57,203 INFO [RpcServer.priority.RWQ.Fifo.write.handler=0,queue=0,port=40189 {}] master.ServerManager(517): Registering regionserver=9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:57,211 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1440): Config from master: hbase.rootdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:39:57,212 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1440): Config from master: fs.defaultFS=hdfs://localhost:40229 2024-11-23T05:39:57,212 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1440): Config from master: hbase.master.info.port=-1 2024-11-23T05:39:57,235 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:39:57,236 DEBUG [RS:0;9abfc7c2569e:45719 {}] zookeeper.ZKUtil(111): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on existing znode=/hbase/rs/9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:57,237 WARN [RS:0;9abfc7c2569e:45719 {}] hbase.ZNodeClearer(65): Environment variable HBASE_ZNODE_FILE not set; znodes will not be cleared on crash by start scripts (Longer MTTR!) 2024-11-23T05:39:57,237 INFO [RS:0;9abfc7c2569e:45719 {}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:39:57,237 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1793): logDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/WALs/9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:57,239 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(179): RegionServer ephemeral node created, adding [9abfc7c2569e,45719,1732340394120] 2024-11-23T05:39:57,266 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.MetricsRegionServerWrapperImpl(120): Computing regionserver metrics every 5000 milliseconds 2024-11-23T05:39:57,281 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.MemStoreFlusher(131): globalMemStoreLimit=880 M, globalMemStoreLimitLowMark=836 M, Offheap=false 2024-11-23T05:39:57,285 INFO [RS:0;9abfc7c2569e:45719 {}] throttle.PressureAwareCompactionThroughputController(131): Compaction throughput configurations, higher bound: 100.00 MB/second, lower bound 50.00 MB/second, off peak: unlimited, tuning period: 60000 ms 2024-11-23T05:39:57,285 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,286 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer$CompactionChecker(1680): CompactionChecker runs every PT1S 2024-11-23T05:39:57,291 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ExecutorStatusChore(48): ExecutorStatusChore runs every 1mins, 0sec 2024-11-23T05:39:57,292 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactedHFilesCleaner, period=120000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,292 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,292 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_OPEN_PRIORITY_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_REGION-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLOSE_META-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_LOG_REPLAY_OPS-regionserver/9abfc7c2569e:0, corePoolSize=2, maxPoolSize=2 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_REGION_REPLICA_FLUSH_OPS-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_REFRESH_PEER-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,293 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_REPLAY_SYNC_REPLICATION_WAL-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,294 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_SWITCH_RPC_THROTTLE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,294 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_CLAIM_REPLICATION_QUEUE-regionserver/9abfc7c2569e:0, corePoolSize=1, maxPoolSize=1 2024-11-23T05:39:57,294 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_SNAPSHOT_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:39:57,294 DEBUG [RS:0;9abfc7c2569e:45719 {}] executor.ExecutorService(95): Starting executor service name=RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0, corePoolSize=3, maxPoolSize=3 2024-11-23T05:39:57,295 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=CompactionChecker, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,295 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=MemstoreFlusherChore, period=1000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,295 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ExecutorStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,295 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=nonceCleaner, period=360000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,295 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=BrokenStoreFileCleaner, period=21600000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,295 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,45719,1732340394120-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:39:57,313 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HeapMemoryManager(209): Starting, tuneOn=false 2024-11-23T05:39:57,315 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,45719,1732340394120-HeapMemoryTunerChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,315 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,315 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.Replication(171): 9abfc7c2569e,45719,1732340394120 started 2024-11-23T05:39:57,332 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(168): Chore ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:57,332 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1482): Serving as 9abfc7c2569e,45719,1732340394120, RpcServer on 9abfc7c2569e/172.17.0.2:45719, sessionid=0x101677e349d0001 2024-11-23T05:39:57,333 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure flush-table-proc starting 2024-11-23T05:39:57,333 DEBUG [RS:0;9abfc7c2569e:45719 {}] flush.RegionServerFlushTableProcedureManager(108): Start region server flush procedure manager 9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:57,333 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,45719,1732340394120' 2024-11-23T05:39:57,333 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/flush-table-proc/abort' 2024-11-23T05:39:57,334 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/flush-table-proc/acquired' 2024-11-23T05:39:57,335 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure flush-table-proc started 2024-11-23T05:39:57,335 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(51): Procedure online-snapshot starting 2024-11-23T05:39:57,335 DEBUG [RS:0;9abfc7c2569e:45719 {}] snapshot.RegionServerSnapshotManager(126): Start Snapshot Manager 9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:57,335 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.ZKProcedureMemberRpcs(357): Starting procedure member '9abfc7c2569e,45719,1732340394120' 2024-11-23T05:39:57,335 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.ZKProcedureMemberRpcs(134): Checking for aborted procedures on node: '/hbase/online-snapshot/abort' 2024-11-23T05:39:57,336 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.ZKProcedureMemberRpcs(154): Looking for new procedures under znode:'/hbase/online-snapshot/acquired' 2024-11-23T05:39:57,336 DEBUG [RS:0;9abfc7c2569e:45719 {}] procedure.RegionServerProcedureManagerHost(53): Procedure online-snapshot started 2024-11-23T05:39:57,336 INFO [RS:0;9abfc7c2569e:45719 {}] quotas.RegionServerRpcQuotaManager(64): Quota support disabled 2024-11-23T05:39:57,336 INFO [RS:0;9abfc7c2569e:45719 {}] quotas.RegionServerSpaceQuotaManager(80): Quota support disabled, not starting space quota manager. 2024-11-23T05:39:57,443 INFO [RS:0;9abfc7c2569e:45719 {}] monitor.StreamSlowMonitor(122): New stream slow monitor defaultMonitorName 2024-11-23T05:39:57,447 INFO [RS:0;9abfc7c2569e:45719 {}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9abfc7c2569e%2C45719%2C1732340394120, suffix=, logDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/WALs/9abfc7c2569e,45719,1732340394120, archiveDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/oldWALs, maxLogs=32 2024-11-23T05:39:57,462 DEBUG [RS:0;9abfc7c2569e:45719 {}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/WALs/9abfc7c2569e,45719,1732340394120/9abfc7c2569e%2C45719%2C1732340394120.1732340397450, exclude list is [], retry=0 2024-11-23T05:39:57,466 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36825,DS-60affd37-2a73-4156-b54e-e57fb7a1828d,DISK] 2024-11-23T05:39:57,469 INFO [RS:0;9abfc7c2569e:45719 {}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/WALs/9abfc7c2569e,45719,1732340394120/9abfc7c2569e%2C45719%2C1732340394120.1732340397450 2024-11-23T05:39:57,470 DEBUG [RS:0;9abfc7c2569e:45719 {}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35975:35975)] 2024-11-23T05:39:57,487 INFO [PEWorker-1 {}] util.FSTableDescriptors(163): Updated hbase:meta table descriptor to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/.tabledesc/.tableinfo.0000000001.1321 2024-11-23T05:39:57,488 INFO [PEWorker-1 {}] regionserver.HRegion(7572): creating {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''}, tableDescriptor='hbase:meta', {TABLE_ATTRIBUTES => {IS_META => 'true', coprocessor$1 => '|org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint|536870911|', METADATA => {'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'}, regionDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:39:57,496 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741833_1009 (size=32) 2024-11-23T05:39:57,904 DEBUG [PEWorker-1 {}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:39:57,909 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T05:39:57,911 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T05:39:57,912 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:57,912 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:57,913 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T05:39:57,915 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T05:39:57,915 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:57,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:57,916 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T05:39:57,919 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T05:39:57,919 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:57,920 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:57,921 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T05:39:57,923 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T05:39:57,923 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:57,924 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:57,924 DEBUG [PEWorker-1 {}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T05:39:57,926 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740 2024-11-23T05:39:57,927 DEBUG [PEWorker-1 {}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740 2024-11-23T05:39:57,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T05:39:57,929 DEBUG [PEWorker-1 {}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T05:39:57,930 DEBUG [PEWorker-1 {}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:39:57,932 DEBUG [PEWorker-1 {}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T05:39:57,936 DEBUG [PEWorker-1 {}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:39:57,937 INFO [PEWorker-1 {}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59330136, jitterRate=-0.11591207981109619}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:39:57,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1006): Region open journal for 1588230740: Writing region info on filesystem at 1732340397905Initializing all the Stores at 1732340397908 (+3 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732340397908Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732340397909 (+1 ms)Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340397909Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732340397909Cleaning up temporary data from old regions at 1732340397929 (+20 ms)Region opened successfully at 1732340397941 (+12 ms) 2024-11-23T05:39:57,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T05:39:57,941 INFO [PEWorker-1 {}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T05:39:57,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T05:39:57,941 DEBUG [PEWorker-1 {}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T05:39:57,942 DEBUG [PEWorker-1 {}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T05:39:57,943 INFO [PEWorker-1 {}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T05:39:57,943 DEBUG [PEWorker-1 {}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732340397941Disabling compacts and flushes for region at 1732340397941Disabling writes for close at 1732340397941Writing region close event to WAL at 1732340397942 (+1 ms)Closed at 1732340397943 (+1 ms) 2024-11-23T05:39:57,946 DEBUG [PEWorker-1 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_ASSIGN_META, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:39:57,946 INFO [PEWorker-1 {}] procedure.InitMetaProcedure(108): Going to assign meta 2024-11-23T05:39:57,951 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN}] 2024-11-23T05:39:57,958 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T05:39:57,961 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN; state=OFFLINE, location=null; forceNewPlan=false, retain=false 2024-11-23T05:39:58,118 DEBUG [9abfc7c2569e:40189 {}] assignment.AssignmentManager(2472): Processing assignQueue; systemServersCount=1, allServersCount=1 2024-11-23T05:39:58,132 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:58,137 INFO [PEWorker-2 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9abfc7c2569e,45719,1732340394120, state=OPENING 2024-11-23T05:39:58,202 DEBUG [PEWorker-2 {}] zookeeper.MetaTableLocator(183): hbase:meta region location doesn't exist, create it 2024-11-23T05:39:58,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:58,215 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:39:58,217 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:39:58,218 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:39:58,221 DEBUG [PEWorker-2 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=2, ppid=1, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN 2024-11-23T05:39:58,224 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=3, ppid=2, state=RUNNABLE, hasLock=false; OpenRegionProcedure 1588230740, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:39:58,403 DEBUG [RSProcedureDispatcher-pool-0 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=AdminService, sasl=false 2024-11-23T05:39:58,407 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:45667, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=AdminService 2024-11-23T05:39:58,416 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(132): Open hbase:meta,,1.1588230740 2024-11-23T05:39:58,416 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.WALFactory(196): Instantiating WALProvider of type class org.apache.hadoop.hbase.wal.AsyncFSWALProvider 2024-11-23T05:39:58,417 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] monitor.StreamSlowMonitor(122): New stream slow monitor .meta 2024-11-23T05:39:58,420 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(613): WAL configuration: blocksize=256 MB, rollsize=128 MB, prefix=9abfc7c2569e%2C45719%2C1732340394120.meta, suffix=.meta, logDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/WALs/9abfc7c2569e,45719,1732340394120, archiveDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/oldWALs, maxLogs=32 2024-11-23T05:39:58,435 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] asyncfs.FanOutOneBlockAsyncDFSOutputHelper(524): When create output stream for /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/WALs/9abfc7c2569e,45719,1732340394120/9abfc7c2569e%2C45719%2C1732340394120.meta.1732340398422.meta, exclude list is [], retry=0 2024-11-23T05:39:58,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] asyncfs.FanOutOneBlockAsyncDFSOutputSaslHelper(816): SASL client skipping handshake in unsecured configuration for addr = 127.0.0.1/127.0.0.1, datanodeId = DatanodeInfoWithStorage[127.0.0.1:36825,DS-60affd37-2a73-4156-b54e-e57fb7a1828d,DISK] 2024-11-23T05:39:58,441 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(991): New WAL /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/WALs/9abfc7c2569e,45719,1732340394120/9abfc7c2569e%2C45719%2C1732340394120.meta.1732340398422.meta 2024-11-23T05:39:58,442 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] wal.AbstractFSWAL(1109): Create new AsyncFSWAL writer with pipeline: [(127.0.0.1/127.0.0.1:35975:35975)] 2024-11-23T05:39:58,442 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7752): Opening region: {ENCODED => 1588230740, NAME => 'hbase:meta,,1', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:39:58,444 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] coprocessor.CoprocessorHost(215): Loading coprocessor class org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint with path null and priority 536870911 2024-11-23T05:39:58,446 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(8280): Registered coprocessor service: region=hbase:meta,,1 service=MultiRowMutationService 2024-11-23T05:39:58,450 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.RegionCoprocessorHost(434): Loaded coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint from HTD of hbase:meta successfully. 2024-11-23T05:39:58,454 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table meta 1588230740 2024-11-23T05:39:58,455 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(898): Instantiated hbase:meta,,1.1588230740; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:39:58,455 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7794): checking encryption for 1588230740 2024-11-23T05:39:58,455 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(7797): checking classloading for 1588230740 2024-11-23T05:39:58,458 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family info of region 1588230740 2024-11-23T05:39:58,460 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName info 2024-11-23T05:39:58,460 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:58,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/info, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:58,461 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family ns of region 1588230740 2024-11-23T05:39:58,463 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName ns 2024-11-23T05:39:58,463 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:58,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/ns, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:58,464 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family rep_barrier of region 1588230740 2024-11-23T05:39:58,465 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName rep_barrier 2024-11-23T05:39:58,465 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:58,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/rep_barrier, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:58,466 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family table of region 1588230740 2024-11-23T05:39:58,468 INFO [StoreOpener-1588230740-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 1588230740 columnFamilyName table 2024-11-23T05:39:58,468 DEBUG [StoreOpener-1588230740-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:58,469 INFO [StoreOpener-1588230740-1 {}] regionserver.HStore(327): Store=1588230740/table, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=ROW_INDEX_V1, compression=NONE 2024-11-23T05:39:58,469 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1038): replaying wal for 1588230740 2024-11-23T05:39:58,471 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740 2024-11-23T05:39:58,474 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740 2024-11-23T05:39:58,476 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1048): stopping wal replay for 1588230740 2024-11-23T05:39:58,477 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1060): Cleaning up temporary data for 1588230740 2024-11-23T05:39:58,478 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table hbase:meta descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:39:58,480 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1093): writing seq id for 1588230740 2024-11-23T05:39:58,482 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1114): Opened 1588230740; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=59649823, jitterRate=-0.11114837229251862}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:39:58,483 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 1588230740 2024-11-23T05:39:58,484 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegion(1006): Region open journal for 1588230740: Running coprocessor pre-open hook at 1732340398456Writing region info on filesystem at 1732340398456Initializing all the Stores at 1732340398458 (+2 ms)Instantiating store for column family {NAME => 'info', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732340398458Instantiating store for column family {NAME => 'ns', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732340398458Instantiating store for column family {NAME => 'rep_barrier', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '2147483647', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340398458Instantiating store for column family {NAME => 'table', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '3', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'ROW_INDEX_V1', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROWCOL', IN_MEMORY => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '8192 B (8KB)'} at 1732340398458Cleaning up temporary data from old regions at 1732340398477 (+19 ms)Running coprocessor post-open hooks at 1732340398483 (+6 ms)Region opened successfully at 1732340398484 (+1 ms) 2024-11-23T05:39:58,490 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2236): Post open deploy tasks for hbase:meta,,1.1588230740, pid=3, masterSystemTime=1732340398393 2024-11-23T05:39:58,501 DEBUG [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] regionserver.HRegionServer(2266): Finished post open deploy task for hbase:meta,,1.1588230740 2024-11-23T05:39:58,502 INFO [RS_OPEN_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_META, pid=3}] handler.AssignRegionHandler(153): Opened hbase:meta,,1.1588230740 2024-11-23T05:39:58,504 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=2 updating hbase:meta row=1588230740, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:58,506 INFO [PEWorker-4 {}] zookeeper.MetaTableLocator(171): Setting hbase:meta replicaId=0 location in ZooKeeper as 9abfc7c2569e,45719,1732340394120, state=OPEN 2024-11-23T05:39:58,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:39:58,564 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeDataChanged, state=SyncConnected, path=/hbase/meta-region-server 2024-11-23T05:39:58,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:39:58,564 DEBUG [zk-event-processor-pool-0 {}] hbase.MetaRegionLocationCache(167): Updating meta znode for path /hbase/meta-region-server: CHANGED 2024-11-23T05:39:58,564 DEBUG [PEWorker-4 {}] procedure2.ProcedureFutureUtil(75): The future has completed while adding callback, give up suspending procedure pid=3, ppid=2, state=RUNNABLE, hasLock=true; OpenRegionProcedure 1588230740, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:58,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=3, resume processing ppid=2 2024-11-23T05:39:58,570 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=3, ppid=2, state=SUCCESS, hasLock=false; OpenRegionProcedure 1588230740, server=9abfc7c2569e,45719,1732340394120 in 342 msec 2024-11-23T05:39:58,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=2, resume processing ppid=1 2024-11-23T05:39:58,578 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=2, ppid=1, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=hbase:meta, region=1588230740, ASSIGN in 621 msec 2024-11-23T05:39:58,580 DEBUG [PEWorker-5 {}] procedure.InitMetaProcedure(97): Execute pid=1, state=RUNNABLE:INIT_META_CREATE_NAMESPACES, hasLock=true; InitMetaProcedure table=hbase:meta 2024-11-23T05:39:58,580 INFO [PEWorker-5 {}] procedure.InitMetaProcedure(114): Going to create {NAME => 'default'} and {NAME => 'hbase'} namespaces 2024-11-23T05:39:58,597 DEBUG [PEWorker-5 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:39:58,598 DEBUG [PEWorker-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:39:58,616 DEBUG [PEWorker-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:39:58,619 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:48111, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:39:58,638 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=1, state=SUCCESS, hasLock=false; InitMetaProcedure table=hbase:meta in 1.6510 sec 2024-11-23T05:39:58,639 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1123): Wait for region servers to report in: status=status unset, state=RUNNING, startTime=1732340398638, completionTime=-1 2024-11-23T05:39:58,641 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ServerManager(903): Finished waiting on RegionServer count=1; waited=0ms, expected min=1 server(s), max=1 server(s), master is running 2024-11-23T05:39:58,641 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1764): Joining cluster... 2024-11-23T05:39:58,665 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1776): Number of RegionServers=1 2024-11-23T05:39:58,665 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$RegionInTransitionChore; timeout=60000, timestamp=1732340458665 2024-11-23T05:39:58,665 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] procedure2.TimeoutExecutorThread(82): ADDED pid=-1, state=WAITING_TIMEOUT, hasLock=false; org.apache.hadoop.hbase.master.assignment.AssignmentManager$DeadServerMetricRegionChore; timeout=120000, timestamp=1732340518665 2024-11-23T05:39:58,665 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] assignment.AssignmentManager(1783): Joined the cluster in 23 msec 2024-11-23T05:39:58,667 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,40189,1732340393249-ClusterStatusChore, period=60000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:58,667 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,40189,1732340393249-BalancerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:58,667 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,40189,1732340393249-RegionNormalizerChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:58,669 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=CatalogJanitor-9abfc7c2569e:40189, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:58,669 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=HbckChore-, period=3600000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:58,670 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:58,676 DEBUG [master/9abfc7c2569e:0.Chore.1 {}] janitor.CatalogJanitor(180): 2024-11-23T05:39:58,696 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1239): Master has completed initialization 4.363sec 2024-11-23T05:39:58,697 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] quotas.MasterQuotaManager(97): Quota support disabled 2024-11-23T05:39:58,699 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] slowlog.SlowLogMasterService(57): Slow/Large requests logging to system table hbase:slowlog is disabled. Quitting. 2024-11-23T05:39:58,699 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] waleventtracker.WALEventTrackerTableCreator(75): wal event tracker requests logging to table REPLICATION.WALEVENTTRACKER is disabled. Quitting. 2024-11-23T05:39:58,700 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] master.ReplicationSinkTrackerTableCreator(90): replication sink tracker requests logging to table REPLICATION.SINK_TRACKER is disabled. Quitting. 2024-11-23T05:39:58,700 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] zookeeper.ZKWatcher(271): not a secure deployment, proceeding 2024-11-23T05:39:58,701 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,40189,1732340393249-MobFileCleanerChore, period=86400, unit=SECONDS is enabled. 2024-11-23T05:39:58,701 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,40189,1732340393249-MobFileCompactionChore, period=604800, unit=SECONDS is enabled. 2024-11-23T05:39:58,709 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster {}] master.HMaster(1374): Balancer post startup initialization complete, took 0 seconds 2024-11-23T05:39:58,710 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=RollingUpgradeChore, period=10, unit=SECONDS is enabled. 2024-11-23T05:39:58,710 INFO [master/9abfc7c2569e:0:becomeActiveMaster {}] hbase.ChoreService(168): Chore ScheduledChore name=9abfc7c2569e,40189,1732340393249-OldWALsDirSizeChore, period=300000, unit=MILLISECONDS is enabled. 2024-11-23T05:39:58,776 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4799df63, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:39:58,778 DEBUG [Time-limited test {}] nio.NioEventLoop(110): -Dio.netty.noKeySetOptimization: false 2024-11-23T05:39:58,778 DEBUG [Time-limited test {}] nio.NioEventLoop(111): -Dio.netty.selectorAutoRebuildThreshold: 512 2024-11-23T05:39:58,781 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:39:58,783 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:39:58,789 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:39:58,791 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:39:58,792 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:39:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@305448db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:39:58,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:39:58,796 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:39:58,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:39:58,799 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35004, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:39:58,802 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1f66000e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:39:58,802 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:39:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:39:58,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:39:58,811 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52372, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:39:58,851 INFO [Time-limited test {}] hbase.HBaseTestingUtil(877): Minicluster is up; activeMaster=9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:58,864 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=211, OpenFileDescriptor=441, MaxFileDescriptor=1048576, SystemLoadAverage=313, ProcessCount=11, AvailableMemoryMB=9389 2024-11-23T05:39:58,872 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:39:58,880 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,40189,1732340393249 2024-11-23T05:39:58,882 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@7afa760f 2024-11-23T05:39:58,883 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:39:58,886 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:39:58,888 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:39:58,891 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:39:58,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T05:39:58,902 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:39:58,904 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 4 2024-11-23T05:39:58,905 DEBUG [PEWorker-2 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:39:58,908 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:39:58,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:39:58,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741835_1011 (size=959) 2024-11-23T05:39:59,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:39:59,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:39:59,330 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:39:59,340 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741836_1012 (size=53) 2024-11-23T05:39:59,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:39:59,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:39:59,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 96c10c5615cf153b0faba5173b51e570, disabling compactions & flushes 2024-11-23T05:39:59,743 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:39:59,743 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:39:59,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. after waiting 0 ms 2024-11-23T05:39:59,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:39:59,744 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:39:59,744 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96c10c5615cf153b0faba5173b51e570: Waiting for close lock at 1732340399743Disabling compacts and flushes for region at 1732340399743Disabling writes for close at 1732340399744 (+1 ms)Writing region close event to WAL at 1732340399744Closed at 1732340399744 2024-11-23T05:39:59,750 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:39:59,757 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732340399750"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732340399750"}]},"ts":"1732340399750"} 2024-11-23T05:39:59,761 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:39:59,763 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:39:59,766 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340399763"}]},"ts":"1732340399763"} 2024-11-23T05:39:59,770 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T05:39:59,771 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, ASSIGN}] 2024-11-23T05:39:59,774 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, ASSIGN 2024-11-23T05:39:59,776 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, ASSIGN; state=OFFLINE, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=false 2024-11-23T05:39:59,928 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=96c10c5615cf153b0faba5173b51e570, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:39:59,935 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=5, ppid=4, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, ASSIGN because future has completed 2024-11-23T05:39:59,936 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:40:00,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:40:00,098 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:00,099 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7752): Opening region: {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:40:00,099 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,099 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:40:00,100 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7794): checking encryption for 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,100 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(7797): checking classloading for 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,103 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,107 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96c10c5615cf153b0faba5173b51e570 columnFamilyName A 2024-11-23T05:40:00,107 DEBUG [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:00,108 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] regionserver.HStore(327): Store=96c10c5615cf153b0faba5173b51e570/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:00,108 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,110 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96c10c5615cf153b0faba5173b51e570 columnFamilyName B 2024-11-23T05:40:00,110 DEBUG [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:00,111 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] regionserver.HStore(327): Store=96c10c5615cf153b0faba5173b51e570/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:00,111 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,114 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96c10c5615cf153b0faba5173b51e570 columnFamilyName C 2024-11-23T05:40:00,114 DEBUG [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:00,114 INFO [StoreOpener-96c10c5615cf153b0faba5173b51e570-1 {}] regionserver.HStore(327): Store=96c10c5615cf153b0faba5173b51e570/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:00,115 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1038): replaying wal for 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,116 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,117 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,119 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1048): stopping wal replay for 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,119 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1060): Cleaning up temporary data for 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,120 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:40:00,123 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1093): writing seq id for 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,127 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:40:00,128 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1114): Opened 96c10c5615cf153b0faba5173b51e570; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69257171, jitterRate=0.03201226890087128}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:40:00,128 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:00,129 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegion(1006): Region open journal for 96c10c5615cf153b0faba5173b51e570: Running coprocessor pre-open hook at 1732340400100Writing region info on filesystem at 1732340400100Initializing all the Stores at 1732340400102 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340400102Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340400103 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340400103Cleaning up temporary data from old regions at 1732340400119 (+16 ms)Running coprocessor post-open hooks at 1732340400128 (+9 ms)Region opened successfully at 1732340400129 (+1 ms) 2024-11-23T05:40:00,131 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., pid=6, masterSystemTime=1732340400090 2024-11-23T05:40:00,135 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:00,135 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=6}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:00,136 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=5 updating hbase:meta row=96c10c5615cf153b0faba5173b51e570, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:00,140 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=6, ppid=5, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:40:00,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=6, resume processing ppid=5 2024-11-23T05:40:00,146 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=6, ppid=5, state=SUCCESS, hasLock=false; OpenRegionProcedure 96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 in 206 msec 2024-11-23T05:40:00,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=5, resume processing ppid=4 2024-11-23T05:40:00,151 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=5, ppid=4, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, ASSIGN in 375 msec 2024-11-23T05:40:00,152 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:40:00,153 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340400152"}]},"ts":"1732340400152"} 2024-11-23T05:40:00,156 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T05:40:00,157 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=4, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:40:00,160 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=4, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2630 sec 2024-11-23T05:40:01,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=4 2024-11-23T05:40:01,058 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:01,065 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@39b7cf82, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,065 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,065 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,067 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,067 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72d6336a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,068 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,069 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,070 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35034, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,072 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7d7323ae, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,072 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,074 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,077 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52382, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,080 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:40:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,40189,1732340393249 2024-11-23T05:40:01,081 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@58b5dcfd 2024-11-23T05:40:01,083 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:40:01,085 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:40:01,089 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c22fa37, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,089 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,089 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,090 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,091 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f338c72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,092 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,094 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35062, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,095 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4184659e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,096 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6d7b17c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,096 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,097 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,098 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,099 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a183203, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,100 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,101 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,102 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35086, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,103 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7ef8cf77, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,104 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6068bcec, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,104 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,104 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,105 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,106 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f55d533, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,109 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35106, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,110 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fc8436a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,110 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@11fbf093, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,110 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,111 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,112 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@21d7efe6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,113 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,114 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,115 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35132, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,116 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@239622c3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,117 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64f1c2cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,117 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,117 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,118 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,118 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6830a8bd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,119 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,120 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35152, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@36baf600, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,122 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1eea3ba9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,122 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,123 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,123 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3d2d67dc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,125 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,126 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35166, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,127 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a82162, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,128 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2d6a39a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,128 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,128 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,129 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24aa1c15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,132 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,133 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2fce0532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,134 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@29b7950b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,134 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,134 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,135 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee22a4d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,136 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,137 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,138 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35200, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,139 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@23451635, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b18ef51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:01,140 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:01,141 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:01,141 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@62078f75, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:01,142 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:01,143 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:01,144 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@145f578c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:01,148 DEBUG [Thread-154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,148 DEBUG [Thread-145 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,149 DEBUG [Thread-160 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,149 DEBUG [Thread-156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,149 DEBUG [Thread-158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,151 DEBUG [Thread-147 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,151 DEBUG [Thread-143 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,152 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:01,152 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,153 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,154 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,156 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52398, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,156 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,156 DEBUG [Thread-151 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52406, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,158 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,158 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52414, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,159 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52408, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,159 DEBUG [Thread-149 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:01,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52430, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,160 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,161 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,162 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52446, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,164 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,165 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees 2024-11-23T05:40:01,168 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,170 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:01,172 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,173 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,174 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,176 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=7, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:01,178 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=8, ppid=7, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:01,178 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:01,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T05:40:01,184 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:52464, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:01,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:40:01,259 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:01,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:01,264 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:01,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T05:40:01,366 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:01,376 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,377 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:01,378 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/65835bb3df3344858a0d7e99df4b4ce9 is 50, key is test_row_0/A:col10/1732340401253/Put/seqid=0 2024-11-23T05:40:01,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,378 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,421 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741837_1013 (size=12001) 2024-11-23T05:40:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,455 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340461428, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,463 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340461440, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340461425, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,464 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,465 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340461433, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,466 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,471 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340461458, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,472 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,473 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T05:40:01,548 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:01,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:01,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,585 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340461578, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,586 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340461579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,587 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340461579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,588 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,588 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,589 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340461579, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,589 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340461580, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,590 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,591 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,592 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:01,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:01,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,743 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340461797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,800 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,801 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340461798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,803 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,803 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340461798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340461798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,804 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T05:40:01,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:01,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340461799, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:01,809 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:01,823 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/65835bb3df3344858a0d7e99df4b4ce9 2024-11-23T05:40:01,904 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:01,905 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,905 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:01,991 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/b6c143faa8184e83bba42104b0b6add4 is 50, key is test_row_0/B:col10/1732340401253/Put/seqid=0 2024-11-23T05:40:02,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741838_1014 (size=12001) 2024-11-23T05:40:02,051 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/b6c143faa8184e83bba42104b0b6add4 2024-11-23T05:40:02,064 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:02,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,109 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340462108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340462108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/628da39df3c64ffda6fdf75a9324cc70 is 50, key is test_row_0/C:col10/1732340401253/Put/seqid=0 2024-11-23T05:40:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,113 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,114 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340462108, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,116 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340462117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,120 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,121 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,122 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340462117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,124 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,162 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741839_1015 (size=12001) 2024-11-23T05:40:02,223 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:02,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,224 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,224 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T05:40:02,381 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:02,381 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,543 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:02,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,543 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,564 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/628da39df3c64ffda6fdf75a9324cc70 2024-11-23T05:40:02,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/65835bb3df3344858a0d7e99df4b4ce9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/65835bb3df3344858a0d7e99df4b4ce9 2024-11-23T05:40:02,594 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/65835bb3df3344858a0d7e99df4b4ce9, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T05:40:02,601 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/b6c143faa8184e83bba42104b0b6add4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b6c143faa8184e83bba42104b0b6add4 2024-11-23T05:40:02,632 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340462617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,634 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340462627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,634 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,635 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,635 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340462627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,638 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,639 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b6c143faa8184e83bba42104b0b6add4, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T05:40:02,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340462628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,639 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,640 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,641 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/628da39df3c64ffda6fdf75a9324cc70 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/628da39df3c64ffda6fdf75a9324cc70 2024-11-23T05:40:02,655 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/628da39df3c64ffda6fdf75a9324cc70, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T05:40:02,670 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 96c10c5615cf153b0faba5173b51e570 in 1398ms, sequenceid=14, compaction requested=false 2024-11-23T05:40:02,672 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T05:40:02,684 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:02,686 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:02,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:40:02,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:02,698 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:02,698 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,699 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,699 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,706 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/24d0ea93ccfd40a69fd35103a3769928 is 50, key is test_row_0/A:col10/1732340402682/Put/seqid=0 2024-11-23T05:40:02,728 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741840_1016 (size=16681) 2024-11-23T05:40:02,730 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/24d0ea93ccfd40a69fd35103a3769928 2024-11-23T05:40:02,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/4b9d198999a740bba209c38607541260 is 50, key is test_row_0/B:col10/1732340402682/Put/seqid=0 2024-11-23T05:40:02,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741841_1017 (size=12001) 2024-11-23T05:40:02,781 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/4b9d198999a740bba209c38607541260 2024-11-23T05:40:02,814 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340462808, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,815 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d25d22460b0a4725a5190a64ef28d490 is 50, key is test_row_0/C:col10/1732340402682/Put/seqid=0 2024-11-23T05:40:02,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741842_1018 (size=12001) 2024-11-23T05:40:02,851 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=39 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d25d22460b0a4725a5190a64ef28d490 2024-11-23T05:40:02,855 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:02,856 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] handler.RSProcedureHandler(58): pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,856 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=8 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=8 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:02,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/24d0ea93ccfd40a69fd35103a3769928 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/24d0ea93ccfd40a69fd35103a3769928 2024-11-23T05:40:02,894 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/24d0ea93ccfd40a69fd35103a3769928, entries=250, sequenceid=39, filesize=16.3 K 2024-11-23T05:40:02,896 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/4b9d198999a740bba209c38607541260 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b9d198999a740bba209c38607541260 2024-11-23T05:40:02,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b9d198999a740bba209c38607541260, entries=150, sequenceid=39, filesize=11.7 K 2024-11-23T05:40:02,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d25d22460b0a4725a5190a64ef28d490 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d25d22460b0a4725a5190a64ef28d490 2024-11-23T05:40:02,944 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:02,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340462927, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:02,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:02,945 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:02,961 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d25d22460b0a4725a5190a64ef28d490, entries=150, sequenceid=39, filesize=11.7 K 2024-11-23T05:40:02,963 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 96c10c5615cf153b0faba5173b51e570 in 276ms, sequenceid=39, compaction requested=false 2024-11-23T05:40:02,963 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:03,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=8 2024-11-23T05:40:03,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:03,014 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:03,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/0abe77be8a0a4290821050b5c6aead9c is 50, key is test_row_0/A:col10/1732340402759/Put/seqid=0 2024-11-23T05:40:03,071 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741843_1019 (size=12001) 2024-11-23T05:40:03,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:03,161 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:03,261 DEBUG [HBase-Metrics2-1 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'hbase:meta' 2024-11-23T05:40:03,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T05:40:03,329 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:03,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340463326, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,331 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:03,439 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:03,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340463437, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,440 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:03,471 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/0abe77be8a0a4290821050b5c6aead9c 2024-11-23T05:40:03,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1bc4aa544a714273b2483e1812a6c4de is 50, key is test_row_0/B:col10/1732340402759/Put/seqid=0 2024-11-23T05:40:03,545 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741844_1020 (size=12001) 2024-11-23T05:40:03,547 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1bc4aa544a714273b2483e1812a6c4de 2024-11-23T05:40:03,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/9737425bfa2041ffa80fed98d738ae31 is 50, key is test_row_0/C:col10/1732340402759/Put/seqid=0 2024-11-23T05:40:03,616 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741845_1021 (size=12001) 2024-11-23T05:40:03,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/9737425bfa2041ffa80fed98d738ae31 2024-11-23T05:40:03,638 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/0abe77be8a0a4290821050b5c6aead9c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0abe77be8a0a4290821050b5c6aead9c 2024-11-23T05:40:03,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:03,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340463648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:03,651 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:03,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340463648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:03,652 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,652 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:03,653 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340463648, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:03,653 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:03,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340463647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:03,654 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2227 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:03,654 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2228 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:03,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:03,656 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,657 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:03,659 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0abe77be8a0a4290821050b5c6aead9c, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T05:40:03,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1bc4aa544a714273b2483e1812a6c4de as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1bc4aa544a714273b2483e1812a6c4de 2024-11-23T05:40:03,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:03,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340463657, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:03,668 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2236 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:03,669 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:03,699 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1bc4aa544a714273b2483e1812a6c4de, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T05:40:03,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/9737425bfa2041ffa80fed98d738ae31 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9737425bfa2041ffa80fed98d738ae31 2024-11-23T05:40:03,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9737425bfa2041ffa80fed98d738ae31, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T05:40:03,737 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 96c10c5615cf153b0faba5173b51e570 in 722ms, sequenceid=50, compaction requested=true 2024-11-23T05:40:03,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:03,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:03,737 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=8}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=8 2024-11-23T05:40:03,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=8 2024-11-23T05:40:03,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=8, resume processing ppid=7 2024-11-23T05:40:03,750 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=8, ppid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5640 sec 2024-11-23T05:40:03,760 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=7, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=7, table=TestAcidGuarantees in 2.5990 sec 2024-11-23T05:40:03,967 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:40:03,976 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:03,976 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:40:03,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:03,993 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/667618133cf8438999926917649f917f is 50, key is test_row_0/A:col10/1732340403969/Put/seqid=0 2024-11-23T05:40:04,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741846_1022 (size=14341) 2024-11-23T05:40:04,034 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/667618133cf8438999926917649f917f 2024-11-23T05:40:04,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:04,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340464037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:04,050 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:04,054 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/815b3dd454864b1e82411c8255c95b6a is 50, key is test_row_0/B:col10/1732340403969/Put/seqid=0 2024-11-23T05:40:04,062 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_hbase_table_meta 2024-11-23T05:40:04,063 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_hbase_table_meta Metrics about Tables on a single HBase RegionServer 2024-11-23T05:40:04,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T05:40:04,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=Coprocessor.Region.CP_org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint Metrics about HBase RegionObservers 2024-11-23T05:40:04,067 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_master_table_store 2024-11-23T05:40:04,067 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_master_table_store Metrics about Tables on a single HBase RegionServer 2024-11-23T05:40:04,072 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T05:40:04,072 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T05:40:04,079 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741847_1023 (size=12001) 2024-11-23T05:40:04,082 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/815b3dd454864b1e82411c8255c95b6a 2024-11-23T05:40:04,102 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/aa05e380d7da4c9bbddeb0b263e326b7 is 50, key is test_row_0/C:col10/1732340403969/Put/seqid=0 2024-11-23T05:40:04,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741848_1024 (size=12001) 2024-11-23T05:40:04,162 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:04,162 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340464156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:04,163 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:04,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:04,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340464366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:04,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:04,531 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/aa05e380d7da4c9bbddeb0b263e326b7 2024-11-23T05:40:04,548 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/667618133cf8438999926917649f917f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/667618133cf8438999926917649f917f 2024-11-23T05:40:04,561 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/667618133cf8438999926917649f917f, entries=200, sequenceid=76, filesize=14.0 K 2024-11-23T05:40:04,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/815b3dd454864b1e82411c8255c95b6a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/815b3dd454864b1e82411c8255c95b6a 2024-11-23T05:40:04,580 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/815b3dd454864b1e82411c8255c95b6a, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T05:40:04,582 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/aa05e380d7da4c9bbddeb0b263e326b7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/aa05e380d7da4c9bbddeb0b263e326b7 2024-11-23T05:40:04,602 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/aa05e380d7da4c9bbddeb0b263e326b7, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T05:40:04,603 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 96c10c5615cf153b0faba5173b51e570 in 627ms, sequenceid=76, compaction requested=true 2024-11-23T05:40:04,604 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:04,614 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:04,617 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:04,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:04,618 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:04,620 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:04,623 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:04,625 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:04,626 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:04,626 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b6c143faa8184e83bba42104b0b6add4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b9d198999a740bba209c38607541260, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1bc4aa544a714273b2483e1812a6c4de, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/815b3dd454864b1e82411c8255c95b6a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=46.9 K 2024-11-23T05:40:04,628 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b6c143faa8184e83bba42104b0b6add4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732340401183 2024-11-23T05:40:04,629 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b9d198999a740bba209c38607541260, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732340401424 2024-11-23T05:40:04,630 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1bc4aa544a714273b2483e1812a6c4de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340402759 2024-11-23T05:40:04,630 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 55024 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:04,631 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:04,631 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 815b3dd454864b1e82411c8255c95b6a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340403285 2024-11-23T05:40:04,631 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:04,632 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/65835bb3df3344858a0d7e99df4b4ce9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/24d0ea93ccfd40a69fd35103a3769928, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0abe77be8a0a4290821050b5c6aead9c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/667618133cf8438999926917649f917f] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=53.7 K 2024-11-23T05:40:04,634 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65835bb3df3344858a0d7e99df4b4ce9, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732340401183 2024-11-23T05:40:04,637 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 24d0ea93ccfd40a69fd35103a3769928, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732340401424 2024-11-23T05:40:04,640 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0abe77be8a0a4290821050b5c6aead9c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340402759 2024-11-23T05:40:04,643 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 667618133cf8438999926917649f917f, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340403285 2024-11-23T05:40:04,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:04,695 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:04,696 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:04,703 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#12 average throughput is 0.23 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:04,705 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/22ebf8dd87fb4d0e8f9b2052b62dd3cc is 50, key is test_row_0/B:col10/1732340403969/Put/seqid=0 2024-11-23T05:40:04,714 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/06959fa58c6746f48e7b22e4193b5d84 is 50, key is test_row_0/A:col10/1732340404031/Put/seqid=0 2024-11-23T05:40:04,719 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#13 average throughput is 0.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:04,720 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5531c63f3fd44d7c81ecc5fb6608e2c9 is 50, key is test_row_0/A:col10/1732340403969/Put/seqid=0 2024-11-23T05:40:04,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741849_1025 (size=12139) 2024-11-23T05:40:04,792 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741850_1026 (size=14341) 2024-11-23T05:40:04,793 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/06959fa58c6746f48e7b22e4193b5d84 2024-11-23T05:40:04,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741851_1027 (size=12139) 2024-11-23T05:40:04,820 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/2ef03c8ef844466cb9e085ceb76b3721 is 50, key is test_row_0/B:col10/1732340404031/Put/seqid=0 2024-11-23T05:40:04,831 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5531c63f3fd44d7c81ecc5fb6608e2c9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5531c63f3fd44d7c81ecc5fb6608e2c9 2024-11-23T05:40:04,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741852_1028 (size=12001) 2024-11-23T05:40:04,842 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/2ef03c8ef844466cb9e085ceb76b3721 2024-11-23T05:40:04,875 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 5531c63f3fd44d7c81ecc5fb6608e2c9(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:04,875 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:04,880 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1b605a49168c45ab92cb3f0e4ff2f308 is 50, key is test_row_0/C:col10/1732340404031/Put/seqid=0 2024-11-23T05:40:04,886 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=12, startTime=1732340404605; duration=0sec 2024-11-23T05:40:04,886 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:04,887 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:04,887 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:04,890 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:04,890 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:04,890 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:04,891 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/628da39df3c64ffda6fdf75a9324cc70, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d25d22460b0a4725a5190a64ef28d490, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9737425bfa2041ffa80fed98d738ae31, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/aa05e380d7da4c9bbddeb0b263e326b7] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=46.9 K 2024-11-23T05:40:04,892 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 628da39df3c64ffda6fdf75a9324cc70, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732340401183 2024-11-23T05:40:04,898 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d25d22460b0a4725a5190a64ef28d490, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=39, earliestPutTs=1732340401424 2024-11-23T05:40:04,899 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9737425bfa2041ffa80fed98d738ae31, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340402759 2024-11-23T05:40:04,901 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa05e380d7da4c9bbddeb0b263e326b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340403285 2024-11-23T05:40:04,917 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741853_1029 (size=12001) 2024-11-23T05:40:04,922 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=87 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1b605a49168c45ab92cb3f0e4ff2f308 2024-11-23T05:40:04,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/06959fa58c6746f48e7b22e4193b5d84 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/06959fa58c6746f48e7b22e4193b5d84 2024-11-23T05:40:04,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/06959fa58c6746f48e7b22e4193b5d84, entries=200, sequenceid=87, filesize=14.0 K 2024-11-23T05:40:04,951 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/2ef03c8ef844466cb9e085ceb76b3721 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/2ef03c8ef844466cb9e085ceb76b3721 2024-11-23T05:40:04,964 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#17 average throughput is 0.82 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:04,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/2ef03c8ef844466cb9e085ceb76b3721, entries=150, sequenceid=87, filesize=11.7 K 2024-11-23T05:40:04,965 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1d4f981d18d3442dbcd819de8f5e949e is 50, key is test_row_0/C:col10/1732340403969/Put/seqid=0 2024-11-23T05:40:04,966 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1b605a49168c45ab92cb3f0e4ff2f308 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1b605a49168c45ab92cb3f0e4ff2f308 2024-11-23T05:40:04,991 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741854_1030 (size=12139) 2024-11-23T05:40:04,992 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1b605a49168c45ab92cb3f0e4ff2f308, entries=150, sequenceid=87, filesize=11.7 K 2024-11-23T05:40:04,996 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=100.63 KB/103050 for 96c10c5615cf153b0faba5173b51e570 in 301ms, sequenceid=87, compaction requested=false 2024-11-23T05:40:04,996 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:05,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:05,004 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:05,004 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T05:40:05,004 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1d4f981d18d3442dbcd819de8f5e949e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1d4f981d18d3442dbcd819de8f5e949e 2024-11-23T05:40:05,027 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into 1d4f981d18d3442dbcd819de8f5e949e(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:05,027 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:05,027 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=12, startTime=1732340404617; duration=0sec 2024-11-23T05:40:05,027 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:05,027 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:05,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/a22693d019684be992eb1a11cac119cc is 50, key is test_row_0/A:col10/1732340404867/Put/seqid=0 2024-11-23T05:40:05,094 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741855_1031 (size=16681) 2024-11-23T05:40:05,174 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/22ebf8dd87fb4d0e8f9b2052b62dd3cc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/22ebf8dd87fb4d0e8f9b2052b62dd3cc 2024-11-23T05:40:05,195 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 22ebf8dd87fb4d0e8f9b2052b62dd3cc(size=11.9 K), total size for store is 23.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:05,195 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:05,195 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=12, startTime=1732340404617; duration=0sec 2024-11-23T05:40:05,195 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:05,195 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:05,209 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340465200, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340465318, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,320 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=7 2024-11-23T05:40:05,338 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:05,338 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:05,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees 2024-11-23T05:40:05,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T05:40:05,342 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:05,343 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=9, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:05,344 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=10, ppid=9, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:05,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T05:40:05,495 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/a22693d019684be992eb1a11cac119cc 2024-11-23T05:40:05,500 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:05,500 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:05,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,501 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,501 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,529 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6074f69ac8c140f8bafbd1bab5701168 is 50, key is test_row_0/B:col10/1732340404867/Put/seqid=0 2024-11-23T05:40:05,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340465526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741856_1032 (size=12001) 2024-11-23T05:40:05,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T05:40:05,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:05,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,659 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,659 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,670 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340465667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,671 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340465667, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,671 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4248 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,672 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,672 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4246 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,673 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,693 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340465686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,694 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4237 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:05,695 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,697 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340465687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,698 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4266 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,699 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,814 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,815 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,815 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:05,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340465847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:05,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:05,850 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:05,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T05:40:05,970 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:05,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:05,970 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,971 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:05,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,006 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6074f69ac8c140f8bafbd1bab5701168 2024-11-23T05:40:06,044 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/f339465b0dfd433b8697251c2512d1d5 is 50, key is test_row_0/C:col10/1732340404867/Put/seqid=0 2024-11-23T05:40:06,098 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741857_1033 (size=12001) 2024-11-23T05:40:06,126 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:06,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:06,126 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,126 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,127 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,282 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,283 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,283 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,359 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340466356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:06,437 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:06,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:06,438 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,438 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] handler.RSProcedureHandler(58): pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,439 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=10 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=10 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:06,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T05:40:06,500 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=107 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/f339465b0dfd433b8697251c2512d1d5 2024-11-23T05:40:06,523 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/a22693d019684be992eb1a11cac119cc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a22693d019684be992eb1a11cac119cc 2024-11-23T05:40:06,549 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a22693d019684be992eb1a11cac119cc, entries=250, sequenceid=107, filesize=16.3 K 2024-11-23T05:40:06,552 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6074f69ac8c140f8bafbd1bab5701168 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6074f69ac8c140f8bafbd1bab5701168 2024-11-23T05:40:06,564 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6074f69ac8c140f8bafbd1bab5701168, entries=150, sequenceid=107, filesize=11.7 K 2024-11-23T05:40:06,566 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/f339465b0dfd433b8697251c2512d1d5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f339465b0dfd433b8697251c2512d1d5 2024-11-23T05:40:06,588 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f339465b0dfd433b8697251c2512d1d5, entries=150, sequenceid=107, filesize=11.7 K 2024-11-23T05:40:06,591 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 96c10c5615cf153b0faba5173b51e570 in 1586ms, sequenceid=107, compaction requested=true 2024-11-23T05:40:06,591 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:06,592 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:06,592 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:06,592 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:06,593 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:06,594 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 43161 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:06,594 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=10 2024-11-23T05:40:06,595 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:06,595 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:06,595 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:06,595 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,595 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,595 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5531c63f3fd44d7c81ecc5fb6608e2c9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/06959fa58c6746f48e7b22e4193b5d84, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a22693d019684be992eb1a11cac119cc] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=42.1 K 2024-11-23T05:40:06,595 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T05:40:06,595 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,596 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/22ebf8dd87fb4d0e8f9b2052b62dd3cc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/2ef03c8ef844466cb9e085ceb76b3721, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6074f69ac8c140f8bafbd1bab5701168] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=35.3 K 2024-11-23T05:40:06,596 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5531c63f3fd44d7c81ecc5fb6608e2c9, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340403285 2024-11-23T05:40:06,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:06,597 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 22ebf8dd87fb4d0e8f9b2052b62dd3cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340403285 2024-11-23T05:40:06,598 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06959fa58c6746f48e7b22e4193b5d84, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732340404009 2024-11-23T05:40:06,599 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ef03c8ef844466cb9e085ceb76b3721, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732340404009 2024-11-23T05:40:06,596 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:06,600 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a22693d019684be992eb1a11cac119cc, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732340404867 2024-11-23T05:40:06,600 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6074f69ac8c140f8bafbd1bab5701168, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732340404867 2024-11-23T05:40:06,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/04688a2b58764aa19e1713e3e6c095a6 is 50, key is test_row_0/A:col10/1732340405174/Put/seqid=0 2024-11-23T05:40:06,649 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#22 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:06,650 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/5bbb545d126c4b5b81ca1c6e1357ee6c is 50, key is test_row_0/B:col10/1732340404867/Put/seqid=0 2024-11-23T05:40:06,659 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#23 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:06,660 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/192284d824794389b7d4387ccea074d6 is 50, key is test_row_0/A:col10/1732340404867/Put/seqid=0 2024-11-23T05:40:06,721 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741858_1034 (size=12001) 2024-11-23T05:40:06,728 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/04688a2b58764aa19e1713e3e6c095a6 2024-11-23T05:40:06,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741859_1035 (size=12241) 2024-11-23T05:40:06,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741860_1036 (size=12241) 2024-11-23T05:40:06,764 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/5bbb545d126c4b5b81ca1c6e1357ee6c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5bbb545d126c4b5b81ca1c6e1357ee6c 2024-11-23T05:40:06,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1ea8fcc78674478ebc36cc05b3b31d98 is 50, key is test_row_0/B:col10/1732340405174/Put/seqid=0 2024-11-23T05:40:06,775 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/192284d824794389b7d4387ccea074d6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/192284d824794389b7d4387ccea074d6 2024-11-23T05:40:06,783 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 5bbb545d126c4b5b81ca1c6e1357ee6c(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:06,783 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:06,783 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340406592; duration=0sec 2024-11-23T05:40:06,783 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:06,783 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:06,784 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:06,786 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:06,787 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:06,787 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,787 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1d4f981d18d3442dbcd819de8f5e949e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1b605a49168c45ab92cb3f0e4ff2f308, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f339465b0dfd433b8697251c2512d1d5] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=35.3 K 2024-11-23T05:40:06,788 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1d4f981d18d3442dbcd819de8f5e949e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340403285 2024-11-23T05:40:06,789 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b605a49168c45ab92cb3f0e4ff2f308, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=87, earliestPutTs=1732340404009 2024-11-23T05:40:06,790 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting f339465b0dfd433b8697251c2512d1d5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732340404867 2024-11-23T05:40:06,815 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741861_1037 (size=12001) 2024-11-23T05:40:06,816 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1ea8fcc78674478ebc36cc05b3b31d98 2024-11-23T05:40:06,824 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 192284d824794389b7d4387ccea074d6(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:06,824 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:06,824 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340406591; duration=0sec 2024-11-23T05:40:06,825 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:06,825 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:06,840 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#25 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:06,840 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d3ae6a9abb404725a6342bb2321eca99 is 50, key is test_row_0/C:col10/1732340404867/Put/seqid=0 2024-11-23T05:40:06,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d4d4340199834354a8effcce81c3d0aa is 50, key is test_row_0/C:col10/1732340405174/Put/seqid=0 2024-11-23T05:40:06,877 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741862_1038 (size=12241) 2024-11-23T05:40:06,897 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d3ae6a9abb404725a6342bb2321eca99 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d3ae6a9abb404725a6342bb2321eca99 2024-11-23T05:40:06,901 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741863_1039 (size=12001) 2024-11-23T05:40:06,904 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=126 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d4d4340199834354a8effcce81c3d0aa 2024-11-23T05:40:06,910 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into d3ae6a9abb404725a6342bb2321eca99(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:06,910 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:06,910 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=13, startTime=1732340406596; duration=0sec 2024-11-23T05:40:06,911 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:06,911 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:06,915 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/04688a2b58764aa19e1713e3e6c095a6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/04688a2b58764aa19e1713e3e6c095a6 2024-11-23T05:40:06,927 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/04688a2b58764aa19e1713e3e6c095a6, entries=150, sequenceid=126, filesize=11.7 K 2024-11-23T05:40:06,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1ea8fcc78674478ebc36cc05b3b31d98 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1ea8fcc78674478ebc36cc05b3b31d98 2024-11-23T05:40:06,945 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1ea8fcc78674478ebc36cc05b3b31d98, entries=150, sequenceid=126, filesize=11.7 K 2024-11-23T05:40:06,947 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/d4d4340199834354a8effcce81c3d0aa as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d4d4340199834354a8effcce81c3d0aa 2024-11-23T05:40:06,957 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d4d4340199834354a8effcce81c3d0aa, entries=150, sequenceid=126, filesize=11.7 K 2024-11-23T05:40:06,962 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=0 B/0 for 96c10c5615cf153b0faba5173b51e570 in 367ms, sequenceid=126, compaction requested=false 2024-11-23T05:40:06,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:06,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:06,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=10}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=10 2024-11-23T05:40:06,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=10 2024-11-23T05:40:06,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=10, resume processing ppid=9 2024-11-23T05:40:06,972 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=10, ppid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.6210 sec 2024-11-23T05:40:06,975 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=9, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=9, table=TestAcidGuarantees in 1.6340 sec 2024-11-23T05:40:07,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:07,444 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:07,445 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:07,455 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/f682de6fa9454164a5f55e921ac23cff is 50, key is test_row_0/A:col10/1732340407440/Put/seqid=0 2024-11-23T05:40:07,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=9 2024-11-23T05:40:07,497 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:07,498 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:07,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees 2024-11-23T05:40:07,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T05:40:07,505 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:07,508 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=11, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:07,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=12, ppid=11, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:07,511 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741864_1040 (size=14541) 2024-11-23T05:40:07,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/f682de6fa9454164a5f55e921ac23cff 2024-11-23T05:40:07,540 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1fae2934803944ef89f7bd4641ac098f is 50, key is test_row_0/B:col10/1732340407440/Put/seqid=0 2024-11-23T05:40:07,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741865_1041 (size=12151) 2024-11-23T05:40:07,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T05:40:07,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T05:40:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:07,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:07,669 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,738 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:07,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340467730, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:07,739 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:07,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T05:40:07,824 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T05:40:07,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:07,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:07,824 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:07,824 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:07,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 126 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340467847, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:07,849 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:07,979 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T05:40:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:07,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:07,979 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] handler.RSProcedureHandler(58): pid=12 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=12 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:07,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=12 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:08,002 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1fae2934803944ef89f7bd4641ac098f 2024-11-23T05:40:08,014 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4f61b5b6e16c4999b427f94b48a2567a is 50, key is test_row_0/C:col10/1732340407440/Put/seqid=0 2024-11-23T05:40:08,028 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741866_1042 (size=12151) 2024-11-23T05:40:08,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=140 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4f61b5b6e16c4999b427f94b48a2567a 2024-11-23T05:40:08,043 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/f682de6fa9454164a5f55e921ac23cff as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/f682de6fa9454164a5f55e921ac23cff 2024-11-23T05:40:08,052 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/f682de6fa9454164a5f55e921ac23cff, entries=200, sequenceid=140, filesize=14.2 K 2024-11-23T05:40:08,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/1fae2934803944ef89f7bd4641ac098f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1fae2934803944ef89f7bd4641ac098f 2024-11-23T05:40:08,062 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:08,063 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 127 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340468056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:08,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,063 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:08,066 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1fae2934803944ef89f7bd4641ac098f, entries=150, sequenceid=140, filesize=11.9 K 2024-11-23T05:40:08,068 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4f61b5b6e16c4999b427f94b48a2567a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4f61b5b6e16c4999b427f94b48a2567a 2024-11-23T05:40:08,083 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4f61b5b6e16c4999b427f94b48a2567a, entries=150, sequenceid=140, filesize=11.9 K 2024-11-23T05:40:08,089 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 96c10c5615cf153b0faba5173b51e570 in 645ms, sequenceid=140, compaction requested=true 2024-11-23T05:40:08,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:08,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:08,090 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:08,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:08,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:08,090 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:08,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:08,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:08,090 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:08,091 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:08,092 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:08,092 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38783 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:08,092 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:08,092 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:08,092 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:08,092 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5bbb545d126c4b5b81ca1c6e1357ee6c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1ea8fcc78674478ebc36cc05b3b31d98, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1fae2934803944ef89f7bd4641ac098f] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=35.5 K 2024-11-23T05:40:08,092 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/192284d824794389b7d4387ccea074d6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/04688a2b58764aa19e1713e3e6c095a6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/f682de6fa9454164a5f55e921ac23cff] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=37.9 K 2024-11-23T05:40:08,093 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5bbb545d126c4b5b81ca1c6e1357ee6c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732340404867 2024-11-23T05:40:08,093 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 192284d824794389b7d4387ccea074d6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732340404867 2024-11-23T05:40:08,094 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea8fcc78674478ebc36cc05b3b31d98, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732340405164 2024-11-23T05:40:08,094 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 04688a2b58764aa19e1713e3e6c095a6, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732340405164 2024-11-23T05:40:08,095 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1fae2934803944ef89f7bd4641ac098f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732340407425 2024-11-23T05:40:08,095 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f682de6fa9454164a5f55e921ac23cff, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732340407407 2024-11-23T05:40:08,117 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#30 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:08,118 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/a0f11811fb2441d191b13d33309a602a is 50, key is test_row_0/A:col10/1732340407440/Put/seqid=0 2024-11-23T05:40:08,122 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#31 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:08,123 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/10eaed887ee245b19f59d949ef616258 is 50, key is test_row_0/B:col10/1732340407440/Put/seqid=0 2024-11-23T05:40:08,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T05:40:08,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=12 2024-11-23T05:40:08,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:08,134 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:40:08,135 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741867_1043 (size=12493) 2024-11-23T05:40:08,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/3991dcd755904ee2ba09f9cbc44c23b3 is 50, key is test_row_0/A:col10/1732340407704/Put/seqid=0 2024-11-23T05:40:08,150 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/a0f11811fb2441d191b13d33309a602a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a0f11811fb2441d191b13d33309a602a 2024-11-23T05:40:08,162 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into a0f11811fb2441d191b13d33309a602a(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:08,162 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:08,162 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340408090; duration=0sec 2024-11-23T05:40:08,163 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:08,163 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:08,163 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:08,165 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741868_1044 (size=12493) 2024-11-23T05:40:08,166 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:08,166 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:08,166 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:08,166 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d3ae6a9abb404725a6342bb2321eca99, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d4d4340199834354a8effcce81c3d0aa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4f61b5b6e16c4999b427f94b48a2567a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=35.5 K 2024-11-23T05:40:08,168 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3ae6a9abb404725a6342bb2321eca99, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=107, earliestPutTs=1732340404867 2024-11-23T05:40:08,170 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4d4340199834354a8effcce81c3d0aa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=126, earliestPutTs=1732340405164 2024-11-23T05:40:08,171 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f61b5b6e16c4999b427f94b48a2567a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732340407425 2024-11-23T05:40:08,186 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/10eaed887ee245b19f59d949ef616258 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/10eaed887ee245b19f59d949ef616258 2024-11-23T05:40:08,201 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 10eaed887ee245b19f59d949ef616258(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:08,201 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:08,202 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340408090; duration=0sec 2024-11-23T05:40:08,202 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:08,202 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:08,209 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#33 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:08,209 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741869_1045 (size=12151) 2024-11-23T05:40:08,210 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/48ca20cbd601450abf0e30b7dd04c331 is 50, key is test_row_0/C:col10/1732340407440/Put/seqid=0 2024-11-23T05:40:08,247 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741870_1046 (size=12493) 2024-11-23T05:40:08,262 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/48ca20cbd601450abf0e30b7dd04c331 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/48ca20cbd601450abf0e30b7dd04c331 2024-11-23T05:40:08,275 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into 48ca20cbd601450abf0e30b7dd04c331(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:08,275 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:08,275 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=13, startTime=1732340408090; duration=0sec 2024-11-23T05:40:08,276 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:08,276 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:08,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:08,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:08,419 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:08,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340468415, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,420 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:08,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:08,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340468526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:08,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,530 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:08,610 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/3991dcd755904ee2ba09f9cbc44c23b3 2024-11-23T05:40:08,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T05:40:08,641 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6d8fb43288534ecf825ed140d51a0f88 is 50, key is test_row_0/B:col10/1732340407704/Put/seqid=0 2024-11-23T05:40:08,665 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741871_1047 (size=12151) 2024-11-23T05:40:08,666 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6d8fb43288534ecf825ed140d51a0f88 2024-11-23T05:40:08,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4698848bffb945349252373d34942a73 is 50, key is test_row_0/C:col10/1732340407704/Put/seqid=0 2024-11-23T05:40:08,720 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741872_1048 (size=12151) 2024-11-23T05:40:08,723 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=165 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4698848bffb945349252373d34942a73 2024-11-23T05:40:08,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/3991dcd755904ee2ba09f9cbc44c23b3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3991dcd755904ee2ba09f9cbc44c23b3 2024-11-23T05:40:08,739 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:08,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340468738, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:08,741 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:08,750 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3991dcd755904ee2ba09f9cbc44c23b3, entries=150, sequenceid=165, filesize=11.9 K 2024-11-23T05:40:08,753 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6d8fb43288534ecf825ed140d51a0f88 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6d8fb43288534ecf825ed140d51a0f88 2024-11-23T05:40:08,764 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6d8fb43288534ecf825ed140d51a0f88, entries=150, sequenceid=165, filesize=11.9 K 2024-11-23T05:40:08,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4698848bffb945349252373d34942a73 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4698848bffb945349252373d34942a73 2024-11-23T05:40:08,797 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4698848bffb945349252373d34942a73, entries=150, sequenceid=165, filesize=11.9 K 2024-11-23T05:40:08,802 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 96c10c5615cf153b0faba5173b51e570 in 669ms, sequenceid=165, compaction requested=false 2024-11-23T05:40:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:08,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=12}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=12 2024-11-23T05:40:08,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=12 2024-11-23T05:40:08,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=12, resume processing ppid=11 2024-11-23T05:40:08,808 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=12, ppid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.2960 sec 2024-11-23T05:40:08,813 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=11, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=11, table=TestAcidGuarantees in 1.3130 sec 2024-11-23T05:40:09,049 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:09,050 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:40:09,050 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:09,077 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/9ee5705175134eaaa860603dccb86460 is 50, key is test_row_0/A:col10/1732340409048/Put/seqid=0 2024-11-23T05:40:09,117 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741873_1049 (size=14541) 2024-11-23T05:40:09,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340469230, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,234 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,235 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,355 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 162 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340469336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,359 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/9ee5705175134eaaa860603dccb86460 2024-11-23T05:40:09,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/5acff5ea78224276a051717a220f4d8e is 50, key is test_row_0/B:col10/1732340409048/Put/seqid=0 2024-11-23T05:40:09,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741874_1050 (size=12151) 2024-11-23T05:40:09,560 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/5acff5ea78224276a051717a220f4d8e 2024-11-23T05:40:09,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 163 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340469566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,573 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,601 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ebfa8a6183bf46478a8b9d3ff7b483fd is 50, key is test_row_0/C:col10/1732340409048/Put/seqid=0 2024-11-23T05:40:09,646 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741875_1051 (size=12151) 2024-11-23T05:40:09,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=11 2024-11-23T05:40:09,647 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:09,648 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:09,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees 2024-11-23T05:40:09,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T05:40:09,655 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:09,656 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=13, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:09,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=14, ppid=13, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:09,718 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,718 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340469716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340469716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,719 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8292 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,720 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,720 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,723 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,727 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340469726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,729 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340469726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,729 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8297 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,730 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,730 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8273 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,731 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T05:40:09,811 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T05:40:09,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:09,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:09,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:09,812 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:09,812 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:09,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:09,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:09,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 164 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340469879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:09,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:09,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T05:40:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:09,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:09,967 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] handler.RSProcedureHandler(58): pid=14 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:09,968 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=14 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:09,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=14 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:09,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T05:40:10,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ebfa8a6183bf46478a8b9d3ff7b483fd 2024-11-23T05:40:10,055 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/9ee5705175134eaaa860603dccb86460 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9ee5705175134eaaa860603dccb86460 2024-11-23T05:40:10,076 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9ee5705175134eaaa860603dccb86460, entries=200, sequenceid=180, filesize=14.2 K 2024-11-23T05:40:10,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/5acff5ea78224276a051717a220f4d8e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5acff5ea78224276a051717a220f4d8e 2024-11-23T05:40:10,098 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5acff5ea78224276a051717a220f4d8e, entries=150, sequenceid=180, filesize=11.9 K 2024-11-23T05:40:10,100 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ebfa8a6183bf46478a8b9d3ff7b483fd as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ebfa8a6183bf46478a8b9d3ff7b483fd 2024-11-23T05:40:10,111 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ebfa8a6183bf46478a8b9d3ff7b483fd, entries=150, sequenceid=180, filesize=11.9 K 2024-11-23T05:40:10,112 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96c10c5615cf153b0faba5173b51e570 in 1063ms, sequenceid=180, compaction requested=true 2024-11-23T05:40:10,112 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:10,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:10,112 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:10,112 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:10,113 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:10,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:10,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:10,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:10,113 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:10,115 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:10,115 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:10,115 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:10,115 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:10,116 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:10,116 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:10,116 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/10eaed887ee245b19f59d949ef616258, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6d8fb43288534ecf825ed140d51a0f88, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5acff5ea78224276a051717a220f4d8e] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=35.9 K 2024-11-23T05:40:10,117 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a0f11811fb2441d191b13d33309a602a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3991dcd755904ee2ba09f9cbc44c23b3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9ee5705175134eaaa860603dccb86460] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=38.3 K 2024-11-23T05:40:10,122 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=14 2024-11-23T05:40:10,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:10,123 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:40:10,123 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 10eaed887ee245b19f59d949ef616258, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732340407425 2024-11-23T05:40:10,123 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0f11811fb2441d191b13d33309a602a, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732340407425 2024-11-23T05:40:10,124 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6d8fb43288534ecf825ed140d51a0f88, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732340407664 2024-11-23T05:40:10,124 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3991dcd755904ee2ba09f9cbc44c23b3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732340407664 2024-11-23T05:40:10,124 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5acff5ea78224276a051717a220f4d8e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340408395 2024-11-23T05:40:10,126 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9ee5705175134eaaa860603dccb86460, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340408395 2024-11-23T05:40:10,144 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ba043508be164063acf20b108f89e214 is 50, key is test_row_0/A:col10/1732340409200/Put/seqid=0 2024-11-23T05:40:10,156 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#40 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:10,156 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/20523c7e3f554bddb8c7aeba56c98615 is 50, key is test_row_0/B:col10/1732340409048/Put/seqid=0 2024-11-23T05:40:10,173 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#41 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:10,174 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/8d8175401aa048bca76df71b230fa5cf is 50, key is test_row_0/A:col10/1732340409048/Put/seqid=0 2024-11-23T05:40:10,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741877_1053 (size=12595) 2024-11-23T05:40:10,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741876_1052 (size=12151) 2024-11-23T05:40:10,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741878_1054 (size=12595) 2024-11-23T05:40:10,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T05:40:10,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:10,389 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:10,423 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:10,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340470421, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:10,424 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:10,527 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:10,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340470527, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:10,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:10,529 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:10,617 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ba043508be164063acf20b108f89e214 2024-11-23T05:40:10,633 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/8d8175401aa048bca76df71b230fa5cf as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/8d8175401aa048bca76df71b230fa5cf 2024-11-23T05:40:10,635 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/20523c7e3f554bddb8c7aeba56c98615 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/20523c7e3f554bddb8c7aeba56c98615 2024-11-23T05:40:10,648 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 8d8175401aa048bca76df71b230fa5cf(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:10,648 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:10,648 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340410112; duration=0sec 2024-11-23T05:40:10,648 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:10,648 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:10,648 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:10,650 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 20523c7e3f554bddb8c7aeba56c98615(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:10,650 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:10,650 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340410113; duration=0sec 2024-11-23T05:40:10,650 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:10,651 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:10,651 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:10,651 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:10,651 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:10,651 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/48ca20cbd601450abf0e30b7dd04c331, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4698848bffb945349252373d34942a73, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ebfa8a6183bf46478a8b9d3ff7b483fd] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=35.9 K 2024-11-23T05:40:10,652 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 48ca20cbd601450abf0e30b7dd04c331, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=140, earliestPutTs=1732340407425 2024-11-23T05:40:10,653 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4698848bffb945349252373d34942a73, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=165, earliestPutTs=1732340407664 2024-11-23T05:40:10,653 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebfa8a6183bf46478a8b9d3ff7b483fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340408395 2024-11-23T05:40:10,662 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/291ed6b06fdf4d4e9f7719270d300e37 is 50, key is test_row_0/B:col10/1732340409200/Put/seqid=0 2024-11-23T05:40:10,688 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#43 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:10,689 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/853dc829cc214e4485a936423483f6be is 50, key is test_row_0/C:col10/1732340409048/Put/seqid=0 2024-11-23T05:40:10,708 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741879_1055 (size=12151) 2024-11-23T05:40:10,730 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741880_1056 (size=12595) 2024-11-23T05:40:10,736 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:10,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 176 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340470736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:10,737 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:10,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T05:40:11,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:11,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 177 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340471046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:11,047 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:11,109 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/291ed6b06fdf4d4e9f7719270d300e37 2024-11-23T05:40:11,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/cacaeb92cace4222ae8688e88c677cc8 is 50, key is test_row_0/C:col10/1732340409200/Put/seqid=0 2024-11-23T05:40:11,141 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/853dc829cc214e4485a936423483f6be as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/853dc829cc214e4485a936423483f6be 2024-11-23T05:40:11,157 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741881_1057 (size=12151) 2024-11-23T05:40:11,158 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into 853dc829cc214e4485a936423483f6be(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:11,158 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:11,158 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=13, startTime=1732340410113; duration=0sec 2024-11-23T05:40:11,158 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:11,158 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:11,559 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=204 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/cacaeb92cace4222ae8688e88c677cc8 2024-11-23T05:40:11,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:11,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 178 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340471556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:11,570 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:11,571 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:11,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ba043508be164063acf20b108f89e214 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ba043508be164063acf20b108f89e214 2024-11-23T05:40:11,599 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ba043508be164063acf20b108f89e214, entries=150, sequenceid=204, filesize=11.9 K 2024-11-23T05:40:11,601 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/291ed6b06fdf4d4e9f7719270d300e37 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/291ed6b06fdf4d4e9f7719270d300e37 2024-11-23T05:40:11,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/291ed6b06fdf4d4e9f7719270d300e37, entries=150, sequenceid=204, filesize=11.9 K 2024-11-23T05:40:11,622 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/cacaeb92cace4222ae8688e88c677cc8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/cacaeb92cace4222ae8688e88c677cc8 2024-11-23T05:40:11,632 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/cacaeb92cace4222ae8688e88c677cc8, entries=150, sequenceid=204, filesize=11.9 K 2024-11-23T05:40:11,633 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 96c10c5615cf153b0faba5173b51e570 in 1511ms, sequenceid=204, compaction requested=false 2024-11-23T05:40:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:11,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=14}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=14 2024-11-23T05:40:11,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=14 2024-11-23T05:40:11,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=14, resume processing ppid=13 2024-11-23T05:40:11,642 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=14, ppid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9800 sec 2024-11-23T05:40:11,656 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=13, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=13, table=TestAcidGuarantees in 1.9980 sec 2024-11-23T05:40:11,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=13 2024-11-23T05:40:11,807 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:11,807 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:11,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees 2024-11-23T05:40:11,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T05:40:11,813 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:11,815 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=15, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:11,815 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=16, ppid=15, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:11,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T05:40:11,971 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=16 2024-11-23T05:40:11,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:11,972 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:40:11,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/e670b30903184a61984dfbd257e55147 is 50, key is test_row_0/A:col10/1732340410419/Put/seqid=0 2024-11-23T05:40:12,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741882_1058 (size=12151) 2024-11-23T05:40:12,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T05:40:12,421 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/e670b30903184a61984dfbd257e55147 2024-11-23T05:40:12,436 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3fdb6ea6b4e84c8fb4236743e9b860fa is 50, key is test_row_0/B:col10/1732340410419/Put/seqid=0 2024-11-23T05:40:12,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T05:40:12,465 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741883_1059 (size=12151) 2024-11-23T05:40:12,466 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3fdb6ea6b4e84c8fb4236743e9b860fa 2024-11-23T05:40:12,481 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1f3e0abef64b48048ee9199ef196362d is 50, key is test_row_0/C:col10/1732340410419/Put/seqid=0 2024-11-23T05:40:12,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741884_1060 (size=12151) 2024-11-23T05:40:12,502 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1f3e0abef64b48048ee9199ef196362d 2024-11-23T05:40:12,513 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/e670b30903184a61984dfbd257e55147 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/e670b30903184a61984dfbd257e55147 2024-11-23T05:40:12,522 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/e670b30903184a61984dfbd257e55147, entries=150, sequenceid=219, filesize=11.9 K 2024-11-23T05:40:12,524 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3fdb6ea6b4e84c8fb4236743e9b860fa as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3fdb6ea6b4e84c8fb4236743e9b860fa 2024-11-23T05:40:12,532 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3fdb6ea6b4e84c8fb4236743e9b860fa, entries=150, sequenceid=219, filesize=11.9 K 2024-11-23T05:40:12,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/1f3e0abef64b48048ee9199ef196362d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1f3e0abef64b48048ee9199ef196362d 2024-11-23T05:40:12,539 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1f3e0abef64b48048ee9199ef196362d, entries=150, sequenceid=219, filesize=11.9 K 2024-11-23T05:40:12,540 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 96c10c5615cf153b0faba5173b51e570 in 568ms, sequenceid=219, compaction requested=true 2024-11-23T05:40:12,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:12,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:12,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=16}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=16 2024-11-23T05:40:12,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=16 2024-11-23T05:40:12,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=16, resume processing ppid=15 2024-11-23T05:40:12,545 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=16, ppid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 727 msec 2024-11-23T05:40:12,549 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=15, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=15, table=TestAcidGuarantees in 740 msec 2024-11-23T05:40:12,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:12,610 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:12,611 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:12,617 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/b845bd9a26454c3ca3027bc8515d0c37 is 50, key is test_row_0/A:col10/1732340412605/Put/seqid=0 2024-11-23T05:40:12,656 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741885_1061 (size=12147) 2024-11-23T05:40:12,762 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:12,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 209 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340472759, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:12,763 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:12,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:12,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 210 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340472866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:12,872 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:12,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:12,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=15 2024-11-23T05:40:12,946 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:12,947 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:12,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees 2024-11-23T05:40:12,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T05:40:12,949 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:12,952 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=17, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:12,952 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=18, ppid=17, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:13,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T05:40:13,057 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/b845bd9a26454c3ca3027bc8515d0c37 2024-11-23T05:40:13,067 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6753b46abd2342f39b22c3c9421833b2 is 50, key is test_row_0/B:col10/1732340412605/Put/seqid=0 2024-11-23T05:40:13,083 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:13,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 211 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340473076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:13,085 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:13,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741886_1062 (size=9757) 2024-11-23T05:40:13,089 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6753b46abd2342f39b22c3c9421833b2 2024-11-23T05:40:13,105 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T05:40:13,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:13,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,105 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,106 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,116 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/2808b2c0e1574eccaf4c387f46dd5394 is 50, key is test_row_0/C:col10/1732340412605/Put/seqid=0 2024-11-23T05:40:13,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741887_1063 (size=9757) 2024-11-23T05:40:13,258 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T05:40:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,259 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T05:40:13,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:13,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 212 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340473396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:13,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:13,397 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:13,412 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T05:40:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,412 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,413 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,538 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=230 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/2808b2c0e1574eccaf4c387f46dd5394 2024-11-23T05:40:13,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/b845bd9a26454c3ca3027bc8515d0c37 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b845bd9a26454c3ca3027bc8515d0c37 2024-11-23T05:40:13,553 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b845bd9a26454c3ca3027bc8515d0c37, entries=150, sequenceid=230, filesize=11.9 K 2024-11-23T05:40:13,555 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/6753b46abd2342f39b22c3c9421833b2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6753b46abd2342f39b22c3c9421833b2 2024-11-23T05:40:13,566 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T05:40:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,566 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] handler.RSProcedureHandler(58): pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,566 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=18 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,567 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6753b46abd2342f39b22c3c9421833b2, entries=100, sequenceid=230, filesize=9.5 K 2024-11-23T05:40:13,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=18 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:13,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T05:40:13,585 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/2808b2c0e1574eccaf4c387f46dd5394 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/2808b2c0e1574eccaf4c387f46dd5394 2024-11-23T05:40:13,592 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/2808b2c0e1574eccaf4c387f46dd5394, entries=100, sequenceid=230, filesize=9.5 K 2024-11-23T05:40:13,594 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 96c10c5615cf153b0faba5173b51e570 in 984ms, sequenceid=230, compaction requested=true 2024-11-23T05:40:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:13,594 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:13,594 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:13,594 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:13,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:13,595 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:13,596 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:13,596 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:13,596 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,597 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/20523c7e3f554bddb8c7aeba56c98615, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/291ed6b06fdf4d4e9f7719270d300e37, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3fdb6ea6b4e84c8fb4236743e9b860fa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6753b46abd2342f39b22c3c9421833b2] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=45.6 K 2024-11-23T05:40:13,597 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49044 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:13,597 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:13,597 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,597 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/8d8175401aa048bca76df71b230fa5cf, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ba043508be164063acf20b108f89e214, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/e670b30903184a61984dfbd257e55147, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b845bd9a26454c3ca3027bc8515d0c37] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=47.9 K 2024-11-23T05:40:13,598 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 20523c7e3f554bddb8c7aeba56c98615, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340408395 2024-11-23T05:40:13,598 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d8175401aa048bca76df71b230fa5cf, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340408395 2024-11-23T05:40:13,598 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 291ed6b06fdf4d4e9f7719270d300e37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732340409200 2024-11-23T05:40:13,599 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3fdb6ea6b4e84c8fb4236743e9b860fa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732340410408 2024-11-23T05:40:13,599 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba043508be164063acf20b108f89e214, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732340409200 2024-11-23T05:40:13,599 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6753b46abd2342f39b22c3c9421833b2, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732340412605 2024-11-23T05:40:13,599 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e670b30903184a61984dfbd257e55147, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732340410408 2024-11-23T05:40:13,600 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b845bd9a26454c3ca3027bc8515d0c37, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732340412598 2024-11-23T05:40:13,625 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#51 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:13,626 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/7bf0e4b401cb4bebaa444affcffe7638 is 50, key is test_row_0/B:col10/1732340412605/Put/seqid=0 2024-11-23T05:40:13,631 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#52 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:13,632 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/6e0bf3f328d345a79f00995c38729593 is 50, key is test_row_0/A:col10/1732340412605/Put/seqid=0 2024-11-23T05:40:13,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741888_1064 (size=12731) 2024-11-23T05:40:13,654 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/7bf0e4b401cb4bebaa444affcffe7638 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/7bf0e4b401cb4bebaa444affcffe7638 2024-11-23T05:40:13,663 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 7bf0e4b401cb4bebaa444affcffe7638(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:13,663 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:13,663 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=12, startTime=1732340413594; duration=0sec 2024-11-23T05:40:13,663 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:13,663 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:13,663 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:13,666 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 46654 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:13,667 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:13,667 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,667 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/853dc829cc214e4485a936423483f6be, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/cacaeb92cace4222ae8688e88c677cc8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1f3e0abef64b48048ee9199ef196362d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/2808b2c0e1574eccaf4c387f46dd5394] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=45.6 K 2024-11-23T05:40:13,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741889_1065 (size=12731) 2024-11-23T05:40:13,669 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 853dc829cc214e4485a936423483f6be, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340408395 2024-11-23T05:40:13,673 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting cacaeb92cace4222ae8688e88c677cc8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=204, earliestPutTs=1732340409200 2024-11-23T05:40:13,674 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1f3e0abef64b48048ee9199ef196362d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732340410408 2024-11-23T05:40:13,678 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2808b2c0e1574eccaf4c387f46dd5394, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732340412605 2024-11-23T05:40:13,714 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#53 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:13,717 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/6260fa23691d4064a06ffe295d3fee28 is 50, key is test_row_0/C:col10/1732340412605/Put/seqid=0 2024-11-23T05:40:13,723 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=18 2024-11-23T05:40:13,723 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,723 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:40:13,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/d57f286270f149bea789fb4dcd04dfe5 is 50, key is test_row_0/A:col10/1732340412746/Put/seqid=0 2024-11-23T05:40:13,751 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741890_1066 (size=12731) 2024-11-23T05:40:13,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741891_1067 (size=12151) 2024-11-23T05:40:13,758 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/d57f286270f149bea789fb4dcd04dfe5 2024-11-23T05:40:13,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/e72b238601b844909451117c64fb58c4 is 50, key is test_row_0/B:col10/1732340412746/Put/seqid=0 2024-11-23T05:40:13,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741892_1068 (size=12151) 2024-11-23T05:40:13,798 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/e72b238601b844909451117c64fb58c4 2024-11-23T05:40:13,828 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4cb85109dad44fb1bb58adb48b2394e7 is 50, key is test_row_0/C:col10/1732340412746/Put/seqid=0 2024-11-23T05:40:13,865 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741893_1069 (size=12151) 2024-11-23T05:40:13,865 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=256 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4cb85109dad44fb1bb58adb48b2394e7 2024-11-23T05:40:13,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/d57f286270f149bea789fb4dcd04dfe5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/d57f286270f149bea789fb4dcd04dfe5 2024-11-23T05:40:13,896 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/d57f286270f149bea789fb4dcd04dfe5, entries=150, sequenceid=256, filesize=11.9 K 2024-11-23T05:40:13,903 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/e72b238601b844909451117c64fb58c4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/e72b238601b844909451117c64fb58c4 2024-11-23T05:40:13,918 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/e72b238601b844909451117c64fb58c4, entries=150, sequenceid=256, filesize=11.9 K 2024-11-23T05:40:13,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:13,920 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:13,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4cb85109dad44fb1bb58adb48b2394e7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4cb85109dad44fb1bb58adb48b2394e7 2024-11-23T05:40:13,942 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4cb85109dad44fb1bb58adb48b2394e7, entries=150, sequenceid=256, filesize=11.9 K 2024-11-23T05:40:13,955 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 96c10c5615cf153b0faba5173b51e570 in 231ms, sequenceid=256, compaction requested=false 2024-11-23T05:40:13,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:13,959 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=18}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=18 2024-11-23T05:40:13,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=18 2024-11-23T05:40:13,963 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=18, resume processing ppid=17 2024-11-23T05:40:13,964 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=18, ppid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.0090 sec 2024-11-23T05:40:13,967 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=17, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=17, table=TestAcidGuarantees in 1.0170 sec 2024-11-23T05:40:13,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:13,977 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:13,977 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:13,987 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/218bbddd3c9147f3a2321cc98742a134 is 50, key is test_row_0/A:col10/1732340413968/Put/seqid=0 2024-11-23T05:40:14,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741894_1070 (size=12301) 2024-11-23T05:40:14,024 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/218bbddd3c9147f3a2321cc98742a134 2024-11-23T05:40:14,038 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/b1f4b3af92484903bdb0c0c6df454671 is 50, key is test_row_0/B:col10/1732340413968/Put/seqid=0 2024-11-23T05:40:14,059 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741895_1071 (size=12301) 2024-11-23T05:40:14,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=17 2024-11-23T05:40:14,086 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/6e0bf3f328d345a79f00995c38729593 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6e0bf3f328d345a79f00995c38729593 2024-11-23T05:40:14,087 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:14,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:14,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees 2024-11-23T05:40:14,102 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T05:40:14,102 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:14,104 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=19, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:14,104 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=20, ppid=19, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:14,106 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 6e0bf3f328d345a79f00995c38729593(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:14,106 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:14,106 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=12, startTime=1732340413594; duration=0sec 2024-11-23T05:40:14,107 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:14,107 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:14,141 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:14,141 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 243 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340474138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,142 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:14,163 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/6260fa23691d4064a06ffe295d3fee28 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/6260fa23691d4064a06ffe295d3fee28 2024-11-23T05:40:14,179 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into 6260fa23691d4064a06ffe295d3fee28(size=12.4 K), total size for store is 24.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:14,179 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:14,179 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=12, startTime=1732340413595; duration=0sec 2024-11-23T05:40:14,180 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:14,180 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:14,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T05:40:14,246 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:14,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 244 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340474246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,247 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:14,259 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T05:40:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,260 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,260 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T05:40:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,415 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T05:40:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,459 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:14,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 245 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340474456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,460 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:14,462 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/b1f4b3af92484903bdb0c0c6df454671 2024-11-23T05:40:14,480 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/fece152f9d504ca2ab1cc7f46f00126a is 50, key is test_row_0/C:col10/1732340413968/Put/seqid=0 2024-11-23T05:40:14,509 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741896_1072 (size=12301) 2024-11-23T05:40:14,510 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=267 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/fece152f9d504ca2ab1cc7f46f00126a 2024-11-23T05:40:14,518 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/218bbddd3c9147f3a2321cc98742a134 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/218bbddd3c9147f3a2321cc98742a134 2024-11-23T05:40:14,532 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/218bbddd3c9147f3a2321cc98742a134, entries=150, sequenceid=267, filesize=12.0 K 2024-11-23T05:40:14,534 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/b1f4b3af92484903bdb0c0c6df454671 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b1f4b3af92484903bdb0c0c6df454671 2024-11-23T05:40:14,559 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b1f4b3af92484903bdb0c0c6df454671, entries=150, sequenceid=267, filesize=12.0 K 2024-11-23T05:40:14,561 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/fece152f9d504ca2ab1cc7f46f00126a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/fece152f9d504ca2ab1cc7f46f00126a 2024-11-23T05:40:14,570 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T05:40:14,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:14,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,570 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] handler.RSProcedureHandler(58): pid=20 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=20 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=20 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:14,574 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/fece152f9d504ca2ab1cc7f46f00126a, entries=150, sequenceid=267, filesize=12.0 K 2024-11-23T05:40:14,576 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 96c10c5615cf153b0faba5173b51e570 in 599ms, sequenceid=267, compaction requested=true 2024-11-23T05:40:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:14,577 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:14,577 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:14,577 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:14,580 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:14,580 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:14,580 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:14,580 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:14,580 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,580 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,580 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/7bf0e4b401cb4bebaa444affcffe7638, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/e72b238601b844909451117c64fb58c4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b1f4b3af92484903bdb0c0c6df454671] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.3 K 2024-11-23T05:40:14,580 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6e0bf3f328d345a79f00995c38729593, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/d57f286270f149bea789fb4dcd04dfe5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/218bbddd3c9147f3a2321cc98742a134] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.3 K 2024-11-23T05:40:14,581 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7bf0e4b401cb4bebaa444affcffe7638, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732340410411 2024-11-23T05:40:14,581 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6e0bf3f328d345a79f00995c38729593, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732340410411 2024-11-23T05:40:14,582 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e72b238601b844909451117c64fb58c4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732340412733 2024-11-23T05:40:14,582 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d57f286270f149bea789fb4dcd04dfe5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732340412733 2024-11-23T05:40:14,582 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b1f4b3af92484903bdb0c0c6df454671, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732340413961 2024-11-23T05:40:14,582 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 218bbddd3c9147f3a2321cc98742a134, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732340413961 2024-11-23T05:40:14,601 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#60 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:14,602 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/69fa527ba3a0409991a8ee9d7973c7b9 is 50, key is test_row_0/B:col10/1732340413968/Put/seqid=0 2024-11-23T05:40:14,609 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#61 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:14,609 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ae318036cf40457391f93fe2c490ac3f is 50, key is test_row_0/A:col10/1732340413968/Put/seqid=0 2024-11-23T05:40:14,632 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741898_1074 (size=12983) 2024-11-23T05:40:14,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741897_1073 (size=12983) 2024-11-23T05:40:14,648 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ae318036cf40457391f93fe2c490ac3f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ae318036cf40457391f93fe2c490ac3f 2024-11-23T05:40:14,660 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into ae318036cf40457391f93fe2c490ac3f(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:14,660 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:14,660 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340414577; duration=0sec 2024-11-23T05:40:14,661 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:14,661 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:14,661 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:14,663 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/69fa527ba3a0409991a8ee9d7973c7b9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/69fa527ba3a0409991a8ee9d7973c7b9 2024-11-23T05:40:14,664 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37183 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:14,664 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:14,664 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,665 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/6260fa23691d4064a06ffe295d3fee28, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4cb85109dad44fb1bb58adb48b2394e7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/fece152f9d504ca2ab1cc7f46f00126a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.3 K 2024-11-23T05:40:14,667 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6260fa23691d4064a06ffe295d3fee28, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=230, earliestPutTs=1732340410411 2024-11-23T05:40:14,668 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4cb85109dad44fb1bb58adb48b2394e7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=256, earliestPutTs=1732340412733 2024-11-23T05:40:14,669 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting fece152f9d504ca2ab1cc7f46f00126a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732340413961 2024-11-23T05:40:14,685 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 69fa527ba3a0409991a8ee9d7973c7b9(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:14,685 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:14,685 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340414577; duration=0sec 2024-11-23T05:40:14,686 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:14,686 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:14,708 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#62 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:14,709 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/596b8060689b43db8f8b83afede4a194 is 50, key is test_row_0/C:col10/1732340413968/Put/seqid=0 2024-11-23T05:40:14,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T05:40:14,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=20 2024-11-23T05:40:14,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:14,730 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:40:14,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/bde61d25d5cd4a5ebdca1c1cda695388 is 50, key is test_row_0/A:col10/1732340414097/Put/seqid=0 2024-11-23T05:40:14,747 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741899_1075 (size=12983) 2024-11-23T05:40:14,759 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/596b8060689b43db8f8b83afede4a194 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/596b8060689b43db8f8b83afede4a194 2024-11-23T05:40:14,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:14,771 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:14,772 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into 596b8060689b43db8f8b83afede4a194(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:14,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:14,772 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=13, startTime=1732340414577; duration=0sec 2024-11-23T05:40:14,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:14,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:14,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741900_1076 (size=12301) 2024-11-23T05:40:14,816 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:14,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 254 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340474812, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,817 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:14,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:14,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 255 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340474926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:14,929 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:15,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:15,137 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 256 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340475136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:15,137 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:15,175 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/bde61d25d5cd4a5ebdca1c1cda695388 2024-11-23T05:40:15,187 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/4b1d53ccecf045dbb20ef4c6ed20c511 is 50, key is test_row_0/B:col10/1732340414097/Put/seqid=0 2024-11-23T05:40:15,200 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741901_1077 (size=12301) 2024-11-23T05:40:15,207 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/4b1d53ccecf045dbb20ef4c6ed20c511 2024-11-23T05:40:15,220 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ae31dca65ed94f149c32390c28e23e4e is 50, key is test_row_0/C:col10/1732340414097/Put/seqid=0 2024-11-23T05:40:15,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741902_1078 (size=12301) 2024-11-23T05:40:15,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T05:40:15,446 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:15,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 257 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340475446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:15,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:15,627 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=296 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ae31dca65ed94f149c32390c28e23e4e 2024-11-23T05:40:15,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/bde61d25d5cd4a5ebdca1c1cda695388 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/bde61d25d5cd4a5ebdca1c1cda695388 2024-11-23T05:40:15,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/bde61d25d5cd4a5ebdca1c1cda695388, entries=150, sequenceid=296, filesize=12.0 K 2024-11-23T05:40:15,648 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/4b1d53ccecf045dbb20ef4c6ed20c511 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b1d53ccecf045dbb20ef4c6ed20c511 2024-11-23T05:40:15,656 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b1d53ccecf045dbb20ef4c6ed20c511, entries=150, sequenceid=296, filesize=12.0 K 2024-11-23T05:40:15,660 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ae31dca65ed94f149c32390c28e23e4e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ae31dca65ed94f149c32390c28e23e4e 2024-11-23T05:40:15,667 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ae31dca65ed94f149c32390c28e23e4e, entries=150, sequenceid=296, filesize=12.0 K 2024-11-23T05:40:15,669 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 96c10c5615cf153b0faba5173b51e570 in 940ms, sequenceid=296, compaction requested=false 2024-11-23T05:40:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:15,669 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=20}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=20 2024-11-23T05:40:15,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=20 2024-11-23T05:40:15,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=20, resume processing ppid=19 2024-11-23T05:40:15,674 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=20, ppid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5670 sec 2024-11-23T05:40:15,681 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=19, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=19, table=TestAcidGuarantees in 1.5800 sec 2024-11-23T05:40:15,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:15,984 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:15,984 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:40:16,005 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5216f6755a724ba1ba0b5512a72aa3d7 is 50, key is test_row_0/A:col10/1732340415958/Put/seqid=0 2024-11-23T05:40:16,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741903_1079 (size=12301) 2024-11-23T05:40:16,053 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5216f6755a724ba1ba0b5512a72aa3d7 2024-11-23T05:40:16,081 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/f1002d58ad5e4b70a5df9dc33c97ed39 is 50, key is test_row_0/B:col10/1732340415958/Put/seqid=0 2024-11-23T05:40:16,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741904_1080 (size=12301) 2024-11-23T05:40:16,130 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:16,131 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 280 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340476127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,133 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:16,238 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:16,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 281 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340476236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,239 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:16,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=19 2024-11-23T05:40:16,246 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:16,247 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:16,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees 2024-11-23T05:40:16,249 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:16,250 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=21, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:16,250 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=22, ppid=21, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:16,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T05:40:16,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T05:40:16,402 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T05:40:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,403 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,446 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:16,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 282 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340476446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,447 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:16,491 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/f1002d58ad5e4b70a5df9dc33c97ed39 2024-11-23T05:40:16,500 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/8b4f9c7632394ec4942095a86526133e is 50, key is test_row_0/C:col10/1732340415958/Put/seqid=0 2024-11-23T05:40:16,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741905_1081 (size=12301) 2024-11-23T05:40:16,556 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T05:40:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:16,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T05:40:16,709 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T05:40:16,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,709 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,756 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:16,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 283 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340476756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:16,757 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:16,863 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T05:40:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,863 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] handler.RSProcedureHandler(58): pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=22 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=22 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:16,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T05:40:16,905 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=309 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/8b4f9c7632394ec4942095a86526133e 2024-11-23T05:40:16,911 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5216f6755a724ba1ba0b5512a72aa3d7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5216f6755a724ba1ba0b5512a72aa3d7 2024-11-23T05:40:16,923 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5216f6755a724ba1ba0b5512a72aa3d7, entries=150, sequenceid=309, filesize=12.0 K 2024-11-23T05:40:16,925 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/f1002d58ad5e4b70a5df9dc33c97ed39 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f1002d58ad5e4b70a5df9dc33c97ed39 2024-11-23T05:40:16,931 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f1002d58ad5e4b70a5df9dc33c97ed39, entries=150, sequenceid=309, filesize=12.0 K 2024-11-23T05:40:16,933 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/8b4f9c7632394ec4942095a86526133e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/8b4f9c7632394ec4942095a86526133e 2024-11-23T05:40:16,941 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/8b4f9c7632394ec4942095a86526133e, entries=150, sequenceid=309, filesize=12.0 K 2024-11-23T05:40:16,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96c10c5615cf153b0faba5173b51e570 in 958ms, sequenceid=309, compaction requested=true 2024-11-23T05:40:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:16,942 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:16,942 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:16,943 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:16,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:16,943 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:16,944 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:16,945 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:16,945 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:16,945 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:16,945 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,945 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:16,945 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ae318036cf40457391f93fe2c490ac3f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/bde61d25d5cd4a5ebdca1c1cda695388, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5216f6755a724ba1ba0b5512a72aa3d7] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.7 K 2024-11-23T05:40:16,945 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/69fa527ba3a0409991a8ee9d7973c7b9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b1d53ccecf045dbb20ef4c6ed20c511, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f1002d58ad5e4b70a5df9dc33c97ed39] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.7 K 2024-11-23T05:40:16,945 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 69fa527ba3a0409991a8ee9d7973c7b9, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732340413961 2024-11-23T05:40:16,945 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae318036cf40457391f93fe2c490ac3f, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732340413961 2024-11-23T05:40:16,946 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b1d53ccecf045dbb20ef4c6ed20c511, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732340414078 2024-11-23T05:40:16,946 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting bde61d25d5cd4a5ebdca1c1cda695388, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732340414078 2024-11-23T05:40:16,946 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting f1002d58ad5e4b70a5df9dc33c97ed39, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732340414801 2024-11-23T05:40:16,946 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5216f6755a724ba1ba0b5512a72aa3d7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732340414801 2024-11-23T05:40:16,960 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#69 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:16,961 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3c46115628754e3095e6665113d45413 is 50, key is test_row_0/B:col10/1732340415958/Put/seqid=0 2024-11-23T05:40:16,961 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#70 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:16,962 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/c854b0361646449bbcebeeee878258fe is 50, key is test_row_0/A:col10/1732340415958/Put/seqid=0 2024-11-23T05:40:16,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741906_1082 (size=13085) 2024-11-23T05:40:16,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741907_1083 (size=13085) 2024-11-23T05:40:16,992 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/c854b0361646449bbcebeeee878258fe as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/c854b0361646449bbcebeeee878258fe 2024-11-23T05:40:17,000 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into c854b0361646449bbcebeeee878258fe(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:17,000 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:17,000 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340416942; duration=0sec 2024-11-23T05:40:17,000 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:17,000 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:17,000 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:17,002 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37585 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:17,002 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:17,002 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:17,002 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/596b8060689b43db8f8b83afede4a194, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ae31dca65ed94f149c32390c28e23e4e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/8b4f9c7632394ec4942095a86526133e] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.7 K 2024-11-23T05:40:17,003 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 596b8060689b43db8f8b83afede4a194, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=267, earliestPutTs=1732340413961 2024-11-23T05:40:17,004 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae31dca65ed94f149c32390c28e23e4e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=296, earliestPutTs=1732340414078 2024-11-23T05:40:17,004 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8b4f9c7632394ec4942095a86526133e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732340414801 2024-11-23T05:40:17,014 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#71 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:17,015 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/dfc02ba24e4042cca05586e5f13f4baf is 50, key is test_row_0/C:col10/1732340415958/Put/seqid=0 2024-11-23T05:40:17,017 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=22 2024-11-23T05:40:17,018 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:17,018 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:40:17,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741908_1084 (size=13085) 2024-11-23T05:40:17,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/14df55a31c444fdf8ab0d161ef9fd720 is 50, key is test_row_0/A:col10/1732340416112/Put/seqid=0 2024-11-23T05:40:17,037 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741909_1085 (size=12301) 2024-11-23T05:40:17,040 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/14df55a31c444fdf8ab0d161ef9fd720 2024-11-23T05:40:17,046 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/dfc02ba24e4042cca05586e5f13f4baf as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/dfc02ba24e4042cca05586e5f13f4baf 2024-11-23T05:40:17,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/a6cc857a90a14823b0edf4f51f70ceae is 50, key is test_row_0/B:col10/1732340416112/Put/seqid=0 2024-11-23T05:40:17,071 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into dfc02ba24e4042cca05586e5f13f4baf(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:17,071 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:17,071 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=13, startTime=1732340416942; duration=0sec 2024-11-23T05:40:17,076 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:17,076 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:17,129 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741910_1086 (size=12301) 2024-11-23T05:40:17,133 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/a6cc857a90a14823b0edf4f51f70ceae 2024-11-23T05:40:17,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4eb6f89ff8e34e1786d7e884bc0dc33b is 50, key is test_row_0/C:col10/1732340416112/Put/seqid=0 2024-11-23T05:40:17,163 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741911_1087 (size=12301) 2024-11-23T05:40:17,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:17,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:17,307 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:17,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 293 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340477307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:17,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T05:40:17,407 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3c46115628754e3095e6665113d45413 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3c46115628754e3095e6665113d45413 2024-11-23T05:40:17,415 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 3c46115628754e3095e6665113d45413(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:17,416 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:17,416 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340416942; duration=0sec 2024-11-23T05:40:17,416 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:17,416 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:17,418 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:17,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 294 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340477416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,419 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:17,564 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=334 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4eb6f89ff8e34e1786d7e884bc0dc33b 2024-11-23T05:40:17,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/14df55a31c444fdf8ab0d161ef9fd720 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/14df55a31c444fdf8ab0d161ef9fd720 2024-11-23T05:40:17,587 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/14df55a31c444fdf8ab0d161ef9fd720, entries=150, sequenceid=334, filesize=12.0 K 2024-11-23T05:40:17,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/a6cc857a90a14823b0edf4f51f70ceae as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/a6cc857a90a14823b0edf4f51f70ceae 2024-11-23T05:40:17,604 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/a6cc857a90a14823b0edf4f51f70ceae, entries=150, sequenceid=334, filesize=12.0 K 2024-11-23T05:40:17,605 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4eb6f89ff8e34e1786d7e884bc0dc33b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4eb6f89ff8e34e1786d7e884bc0dc33b 2024-11-23T05:40:17,613 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4eb6f89ff8e34e1786d7e884bc0dc33b, entries=150, sequenceid=334, filesize=12.0 K 2024-11-23T05:40:17,614 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 96c10c5615cf153b0faba5173b51e570 in 596ms, sequenceid=334, compaction requested=false 2024-11-23T05:40:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:17,615 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=22}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=22 2024-11-23T05:40:17,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=22 2024-11-23T05:40:17,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=22, resume processing ppid=21 2024-11-23T05:40:17,623 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=22, ppid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3670 sec 2024-11-23T05:40:17,625 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=21, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=21, table=TestAcidGuarantees in 1.3770 sec 2024-11-23T05:40:17,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:17,628 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:17,628 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:40:17,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/b61470460128492c827935e4dbeb8fe0 is 50, key is test_row_0/A:col10/1732340417304/Put/seqid=0 2024-11-23T05:40:17,686 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741912_1088 (size=14741) 2024-11-23T05:40:17,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/b61470460128492c827935e4dbeb8fe0 2024-11-23T05:40:17,708 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/645aabcabbc04da69c25aee0b4afd4b9 is 50, key is test_row_0/B:col10/1732340417304/Put/seqid=0 2024-11-23T05:40:17,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741913_1089 (size=12301) 2024-11-23T05:40:17,745 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:17,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 316 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340477742, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:17,745 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,746 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:17,858 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:17,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 317 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340477856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:17,862 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:18,068 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:18,068 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 318 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340478066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:18,069 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:18,126 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/645aabcabbc04da69c25aee0b4afd4b9 2024-11-23T05:40:18,137 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/0de548a4ceb1411088ed9021453494bc is 50, key is test_row_0/C:col10/1732340417304/Put/seqid=0 2024-11-23T05:40:18,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741914_1090 (size=12301) 2024-11-23T05:40:18,379 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:18,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 319 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340478376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:18,381 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:18,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=21 2024-11-23T05:40:18,397 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:18,398 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:18,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees 2024-11-23T05:40:18,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T05:40:18,404 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:18,405 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=23, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:18,405 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=24, ppid=23, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:18,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T05:40:18,552 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=349 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/0de548a4ceb1411088ed9021453494bc 2024-11-23T05:40:18,561 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T05:40:18,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:18,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:18,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:18,561 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] handler.RSProcedureHandler(58): pid=24 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:18,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=24 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:18,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=24 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:18,565 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/b61470460128492c827935e4dbeb8fe0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b61470460128492c827935e4dbeb8fe0 2024-11-23T05:40:18,578 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b61470460128492c827935e4dbeb8fe0, entries=200, sequenceid=349, filesize=14.4 K 2024-11-23T05:40:18,579 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/645aabcabbc04da69c25aee0b4afd4b9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/645aabcabbc04da69c25aee0b4afd4b9 2024-11-23T05:40:18,590 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/645aabcabbc04da69c25aee0b4afd4b9, entries=150, sequenceid=349, filesize=12.0 K 2024-11-23T05:40:18,592 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/0de548a4ceb1411088ed9021453494bc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/0de548a4ceb1411088ed9021453494bc 2024-11-23T05:40:18,603 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/0de548a4ceb1411088ed9021453494bc, entries=150, sequenceid=349, filesize=12.0 K 2024-11-23T05:40:18,606 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 96c10c5615cf153b0faba5173b51e570 in 977ms, sequenceid=349, compaction requested=true 2024-11-23T05:40:18,606 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:18,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:18,606 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:18,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:18,606 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:18,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:18,606 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:18,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:18,607 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:18,609 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40127 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:18,609 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:18,609 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:18,610 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/c854b0361646449bbcebeeee878258fe, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/14df55a31c444fdf8ab0d161ef9fd720, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b61470460128492c827935e4dbeb8fe0] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=39.2 K 2024-11-23T05:40:18,612 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c854b0361646449bbcebeeee878258fe, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732340414801 2024-11-23T05:40:18,612 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:18,613 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:18,613 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:18,613 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3c46115628754e3095e6665113d45413, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/a6cc857a90a14823b0edf4f51f70ceae, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/645aabcabbc04da69c25aee0b4afd4b9] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.8 K 2024-11-23T05:40:18,615 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 14df55a31c444fdf8ab0d161ef9fd720, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732340416110 2024-11-23T05:40:18,621 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3c46115628754e3095e6665113d45413, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732340414801 2024-11-23T05:40:18,622 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a6cc857a90a14823b0edf4f51f70ceae, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732340416110 2024-11-23T05:40:18,623 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b61470460128492c827935e4dbeb8fe0, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732340417289 2024-11-23T05:40:18,623 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 645aabcabbc04da69c25aee0b4afd4b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732340417293 2024-11-23T05:40:18,638 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#78 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:18,638 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/d4420e162df54099882773f68d83529b is 50, key is test_row_0/B:col10/1732340417304/Put/seqid=0 2024-11-23T05:40:18,639 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#79 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:18,640 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/0b9f83961e3048a5bfa139e361813f5d is 50, key is test_row_0/A:col10/1732340417304/Put/seqid=0 2024-11-23T05:40:18,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741916_1092 (size=13187) 2024-11-23T05:40:18,648 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741915_1091 (size=13187) 2024-11-23T05:40:18,657 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/0b9f83961e3048a5bfa139e361813f5d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0b9f83961e3048a5bfa139e361813f5d 2024-11-23T05:40:18,658 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/d4420e162df54099882773f68d83529b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/d4420e162df54099882773f68d83529b 2024-11-23T05:40:18,669 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 0b9f83961e3048a5bfa139e361813f5d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:18,669 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:18,669 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340418606; duration=0sec 2024-11-23T05:40:18,671 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:18,671 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:18,671 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:18,672 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into d4420e162df54099882773f68d83529b(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:18,673 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:18,673 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340418606; duration=0sec 2024-11-23T05:40:18,673 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37687 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:18,673 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:18,673 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:18,673 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:18,673 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:18,673 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/dfc02ba24e4042cca05586e5f13f4baf, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4eb6f89ff8e34e1786d7e884bc0dc33b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/0de548a4ceb1411088ed9021453494bc] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.8 K 2024-11-23T05:40:18,674 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting dfc02ba24e4042cca05586e5f13f4baf, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=309, earliestPutTs=1732340414801 2024-11-23T05:40:18,674 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4eb6f89ff8e34e1786d7e884bc0dc33b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=334, earliestPutTs=1732340416110 2024-11-23T05:40:18,675 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0de548a4ceb1411088ed9021453494bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732340417293 2024-11-23T05:40:18,701 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#80 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:18,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/f423eba513f0418dae4698189a32ad0b is 50, key is test_row_0/C:col10/1732340417304/Put/seqid=0 2024-11-23T05:40:18,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T05:40:18,716 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=24 2024-11-23T05:40:18,716 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:18,717 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:40:18,739 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/3feed8686d104d11a2b49b074d09d0e7 is 50, key is test_row_0/A:col10/1732340417741/Put/seqid=0 2024-11-23T05:40:18,761 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741917_1093 (size=13187) 2024-11-23T05:40:18,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741918_1094 (size=12301) 2024-11-23T05:40:18,804 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/3feed8686d104d11a2b49b074d09d0e7 2024-11-23T05:40:18,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/98db027fcc2f4aa3be85e432aafa9ff5 is 50, key is test_row_0/B:col10/1732340417741/Put/seqid=0 2024-11-23T05:40:18,882 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741919_1095 (size=12301) 2024-11-23T05:40:18,884 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/98db027fcc2f4aa3be85e432aafa9ff5 2024-11-23T05:40:18,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:18,888 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:18,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/9b3f591edb784c919cf24e57ef58bef3 is 50, key is test_row_0/C:col10/1732340417741/Put/seqid=0 2024-11-23T05:40:18,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:18,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 330 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340478947, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:18,950 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:18,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741920_1096 (size=12301) 2024-11-23T05:40:18,968 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=374 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/9b3f591edb784c919cf24e57ef58bef3 2024-11-23T05:40:18,982 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/3feed8686d104d11a2b49b074d09d0e7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3feed8686d104d11a2b49b074d09d0e7 2024-11-23T05:40:18,990 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3feed8686d104d11a2b49b074d09d0e7, entries=150, sequenceid=374, filesize=12.0 K 2024-11-23T05:40:18,994 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/98db027fcc2f4aa3be85e432aafa9ff5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/98db027fcc2f4aa3be85e432aafa9ff5 2024-11-23T05:40:19,004 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/98db027fcc2f4aa3be85e432aafa9ff5, entries=150, sequenceid=374, filesize=12.0 K 2024-11-23T05:40:19,007 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/9b3f591edb784c919cf24e57ef58bef3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9b3f591edb784c919cf24e57ef58bef3 2024-11-23T05:40:19,015 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9b3f591edb784c919cf24e57ef58bef3, entries=150, sequenceid=374, filesize=12.0 K 2024-11-23T05:40:19,017 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 96c10c5615cf153b0faba5173b51e570 in 301ms, sequenceid=374, compaction requested=false 2024-11-23T05:40:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=24}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=24 2024-11-23T05:40:19,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=24 2024-11-23T05:40:19,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=24, resume processing ppid=23 2024-11-23T05:40:19,023 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=24, ppid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 615 msec 2024-11-23T05:40:19,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T05:40:19,027 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=23, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=23, table=TestAcidGuarantees in 626 msec 2024-11-23T05:40:19,060 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:19,061 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:40:19,061 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:19,075 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/73eda7338efb44dca987d8290b33f630 is 50, key is test_row_0/A:col10/1732340418945/Put/seqid=0 2024-11-23T05:40:19,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741921_1097 (size=14741) 2024-11-23T05:40:19,131 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/73eda7338efb44dca987d8290b33f630 2024-11-23T05:40:19,144 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/733c75d6f9b3446988bd9f1e69a4163b is 50, key is test_row_0/B:col10/1732340418945/Put/seqid=0 2024-11-23T05:40:19,174 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741922_1098 (size=12301) 2024-11-23T05:40:19,175 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/733c75d6f9b3446988bd9f1e69a4163b 2024-11-23T05:40:19,184 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/f423eba513f0418dae4698189a32ad0b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f423eba513f0418dae4698189a32ad0b 2024-11-23T05:40:19,193 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/83bddaf53206453d8afa5e7434673622 is 50, key is test_row_0/C:col10/1732340418945/Put/seqid=0 2024-11-23T05:40:19,197 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into f423eba513f0418dae4698189a32ad0b(size=12.9 K), total size for store is 24.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:19,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:19,198 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=13, startTime=1732340418607; duration=0sec 2024-11-23T05:40:19,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:19,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:19,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 351 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340479223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,230 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,230 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741923_1099 (size=12301) 2024-11-23T05:40:19,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 352 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340479336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,339 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=23 2024-11-23T05:40:19,537 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:19,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:19,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees 2024-11-23T05:40:19,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T05:40:19,541 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:19,542 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=25, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:19,542 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=26, ppid=25, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:19,549 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 353 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340479546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,550 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,629 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=388 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/83bddaf53206453d8afa5e7434673622 2024-11-23T05:40:19,636 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/73eda7338efb44dca987d8290b33f630 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/73eda7338efb44dca987d8290b33f630 2024-11-23T05:40:19,641 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/73eda7338efb44dca987d8290b33f630, entries=200, sequenceid=388, filesize=14.4 K 2024-11-23T05:40:19,642 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/733c75d6f9b3446988bd9f1e69a4163b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/733c75d6f9b3446988bd9f1e69a4163b 2024-11-23T05:40:19,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T05:40:19,648 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/733c75d6f9b3446988bd9f1e69a4163b, entries=150, sequenceid=388, filesize=12.0 K 2024-11-23T05:40:19,659 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/83bddaf53206453d8afa5e7434673622 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/83bddaf53206453d8afa5e7434673622 2024-11-23T05:40:19,671 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/83bddaf53206453d8afa5e7434673622, entries=150, sequenceid=388, filesize=12.0 K 2024-11-23T05:40:19,673 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 96c10c5615cf153b0faba5173b51e570 in 612ms, sequenceid=388, compaction requested=true 2024-11-23T05:40:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:19,673 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:19,673 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:19,673 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:19,675 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37789 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:19,675 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40229 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:19,675 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:19,675 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:19,675 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:19,675 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:19,675 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0b9f83961e3048a5bfa139e361813f5d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3feed8686d104d11a2b49b074d09d0e7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/73eda7338efb44dca987d8290b33f630] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=39.3 K 2024-11-23T05:40:19,675 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/d4420e162df54099882773f68d83529b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/98db027fcc2f4aa3be85e432aafa9ff5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/733c75d6f9b3446988bd9f1e69a4163b] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=36.9 K 2024-11-23T05:40:19,676 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d4420e162df54099882773f68d83529b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732340417293 2024-11-23T05:40:19,676 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0b9f83961e3048a5bfa139e361813f5d, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732340417293 2024-11-23T05:40:19,676 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 98db027fcc2f4aa3be85e432aafa9ff5, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732340417720 2024-11-23T05:40:19,676 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3feed8686d104d11a2b49b074d09d0e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732340417720 2024-11-23T05:40:19,676 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 733c75d6f9b3446988bd9f1e69a4163b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732340418927 2024-11-23T05:40:19,677 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 73eda7338efb44dca987d8290b33f630, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732340418914 2024-11-23T05:40:19,694 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#87 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:19,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=26 2024-11-23T05:40:19,695 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:19,695 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T05:40:19,697 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/2e8560dfc8be4eebbe52f27f1121819a is 50, key is test_row_0/A:col10/1732340418945/Put/seqid=0 2024-11-23T05:40:19,703 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#88 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:19,704 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/f2d4012309ec454498672d6ef81b2a5d is 50, key is test_row_0/B:col10/1732340418945/Put/seqid=0 2024-11-23T05:40:19,707 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5553693e619242939d72234c8e7b55b9 is 50, key is test_row_0/A:col10/1732340419104/Put/seqid=0 2024-11-23T05:40:19,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741925_1101 (size=13289) 2024-11-23T05:40:19,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741924_1100 (size=13289) 2024-11-23T05:40:19,742 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:19,755 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741926_1102 (size=12301) 2024-11-23T05:40:19,759 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5553693e619242939d72234c8e7b55b9 2024-11-23T05:40:19,769 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/cd51f684d57049a1890a595111de750f is 50, key is test_row_0/B:col10/1732340419104/Put/seqid=0 2024-11-23T05:40:19,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52414 deadline: 1732340479767, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,772 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340479768, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,772 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340479770, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,772 WARN [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96c10c5615cf153b0faba5173b51e570 of TestAcidGuarantees failed, tries = 9, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 18349 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:19,773 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340479772, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,773 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,774 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741927_1103 (size=12301) 2024-11-23T05:40:19,786 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/cd51f684d57049a1890a595111de750f 2024-11-23T05:40:19,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/a4919a238c4f425e931cfc19242bf8e4 is 50, key is test_row_0/C:col10/1732340419104/Put/seqid=0 2024-11-23T05:40:19,813 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741928_1104 (size=12301) 2024-11-23T05:40:19,818 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=411 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/a4919a238c4f425e931cfc19242bf8e4 2024-11-23T05:40:19,825 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5553693e619242939d72234c8e7b55b9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5553693e619242939d72234c8e7b55b9 2024-11-23T05:40:19,833 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5553693e619242939d72234c8e7b55b9, entries=150, sequenceid=411, filesize=12.0 K 2024-11-23T05:40:19,834 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/cd51f684d57049a1890a595111de750f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/cd51f684d57049a1890a595111de750f 2024-11-23T05:40:19,841 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/cd51f684d57049a1890a595111de750f, entries=150, sequenceid=411, filesize=12.0 K 2024-11-23T05:40:19,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/a4919a238c4f425e931cfc19242bf8e4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/a4919a238c4f425e931cfc19242bf8e4 2024-11-23T05:40:19,851 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/a4919a238c4f425e931cfc19242bf8e4, entries=150, sequenceid=411, filesize=12.0 K 2024-11-23T05:40:19,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 96c10c5615cf153b0faba5173b51e570 in 158ms, sequenceid=411, compaction requested=true 2024-11-23T05:40:19,853 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:19,854 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=26}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=26 2024-11-23T05:40:19,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=26 2024-11-23T05:40:19,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T05:40:19,858 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:19,858 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T05:40:19,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:19,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=26, resume processing ppid=25 2024-11-23T05:40:19,867 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=26, ppid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 313 msec 2024-11-23T05:40:19,870 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=25, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=25, table=TestAcidGuarantees in 330 msec 2024-11-23T05:40:19,877 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/6bc6b2a023844aa78e31738a61481ed6 is 50, key is test_row_0/A:col10/1732340419857/Put/seqid=0 2024-11-23T05:40:19,908 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741929_1105 (size=14741) 2024-11-23T05:40:19,910 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 363 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340479903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340479903, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,911 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,911 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340479904, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,912 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:19,915 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:19,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340479910, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:19,916 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,026 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 364 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340480023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340480023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,027 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,027 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340480023, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340480026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,028 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,148 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/2e8560dfc8be4eebbe52f27f1121819a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/2e8560dfc8be4eebbe52f27f1121819a 2024-11-23T05:40:20,148 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/f2d4012309ec454498672d6ef81b2a5d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f2d4012309ec454498672d6ef81b2a5d 2024-11-23T05:40:20,160 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 2e8560dfc8be4eebbe52f27f1121819a(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:20,160 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:20,160 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340419673; duration=0sec 2024-11-23T05:40:20,160 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:20,160 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:20,160 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:20,163 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into f2d4012309ec454498672d6ef81b2a5d(size=13.0 K), total size for store is 25.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:20,164 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:20,164 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340419673; duration=0sec 2024-11-23T05:40:20,164 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:20,164 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:20,165 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:20,165 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:20,165 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,165 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f423eba513f0418dae4698189a32ad0b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9b3f591edb784c919cf24e57ef58bef3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/83bddaf53206453d8afa5e7434673622, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/a4919a238c4f425e931cfc19242bf8e4] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=48.9 K 2024-11-23T05:40:20,166 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f423eba513f0418dae4698189a32ad0b, keycount=150, bloomtype=ROW, size=12.9 K, encoding=NONE, compression=NONE, seqNum=349, earliestPutTs=1732340417293 2024-11-23T05:40:20,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=25 2024-11-23T05:40:20,166 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:20,167 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:20,168 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9b3f591edb784c919cf24e57ef58bef3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=374, earliestPutTs=1732340417720 2024-11-23T05:40:20,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees 2024-11-23T05:40:20,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T05:40:20,169 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:20,170 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=27, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:20,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=28, ppid=27, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:20,172 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 83bddaf53206453d8afa5e7434673622, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732340418927 2024-11-23T05:40:20,173 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a4919a238c4f425e931cfc19242bf8e4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732340419104 2024-11-23T05:40:20,197 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#93 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:20,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/e53e316a8dc741dda3a9e5d002ea29a9 is 50, key is test_row_0/C:col10/1732340419104/Put/seqid=0 2024-11-23T05:40:20,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 365 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340480236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340480236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340480236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,244 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,244 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340480236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,245 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,246 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741930_1106 (size=13323) 2024-11-23T05:40:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,246 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T05:40:20,309 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/6bc6b2a023844aa78e31738a61481ed6 2024-11-23T05:40:20,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T05:40:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:20,324 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/05a704c00f3c4f25b22e186a05582520 is 50, key is test_row_0/B:col10/1732340419857/Put/seqid=0 2024-11-23T05:40:20,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,324 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,369 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741931_1107 (size=12301) 2024-11-23T05:40:20,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T05:40:20,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:20,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,479 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T05:40:20,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340480556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,560 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,560 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,560 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 366 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340480556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340480556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,561 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,561 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:20,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340480556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:20,562 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:20,633 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T05:40:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,634 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,634 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,658 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/e53e316a8dc741dda3a9e5d002ea29a9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e53e316a8dc741dda3a9e5d002ea29a9 2024-11-23T05:40:20,670 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into e53e316a8dc741dda3a9e5d002ea29a9(size=13.0 K), total size for store is 13.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:20,670 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:20,670 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=12, startTime=1732340419673; duration=0sec 2024-11-23T05:40:20,670 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:20,670 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:20,770 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/05a704c00f3c4f25b22e186a05582520 2024-11-23T05:40:20,784 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/b32cbe98e9df414d80c850e2371a6688 is 50, key is test_row_0/C:col10/1732340419857/Put/seqid=0 2024-11-23T05:40:20,795 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T05:40:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,795 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] handler.RSProcedureHandler(58): pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=28 java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T05:40:20,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=28 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:20,834 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741932_1108 (size=12301) 2024-11-23T05:40:20,835 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=426 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/b32cbe98e9df414d80c850e2371a6688 2024-11-23T05:40:20,846 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/6bc6b2a023844aa78e31738a61481ed6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6bc6b2a023844aa78e31738a61481ed6 2024-11-23T05:40:20,857 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6bc6b2a023844aa78e31738a61481ed6, entries=200, sequenceid=426, filesize=14.4 K 2024-11-23T05:40:20,863 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/05a704c00f3c4f25b22e186a05582520 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/05a704c00f3c4f25b22e186a05582520 2024-11-23T05:40:20,875 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/05a704c00f3c4f25b22e186a05582520, entries=150, sequenceid=426, filesize=12.0 K 2024-11-23T05:40:20,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/b32cbe98e9df414d80c850e2371a6688 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/b32cbe98e9df414d80c850e2371a6688 2024-11-23T05:40:20,891 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/b32cbe98e9df414d80c850e2371a6688, entries=150, sequenceid=426, filesize=12.0 K 2024-11-23T05:40:20,892 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 96c10c5615cf153b0faba5173b51e570 in 1034ms, sequenceid=426, compaction requested=true 2024-11-23T05:40:20,892 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:20,893 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:20,893 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:20,893 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:20,895 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40331 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:20,896 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:20,896 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,896 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/2e8560dfc8be4eebbe52f27f1121819a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5553693e619242939d72234c8e7b55b9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6bc6b2a023844aa78e31738a61481ed6] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=39.4 K 2024-11-23T05:40:20,897 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2e8560dfc8be4eebbe52f27f1121819a, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732340418927 2024-11-23T05:40:20,897 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37891 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:20,897 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5553693e619242939d72234c8e7b55b9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732340419104 2024-11-23T05:40:20,897 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:20,897 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,898 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f2d4012309ec454498672d6ef81b2a5d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/cd51f684d57049a1890a595111de750f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/05a704c00f3c4f25b22e186a05582520] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=37.0 K 2024-11-23T05:40:20,898 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6bc6b2a023844aa78e31738a61481ed6, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732340419757 2024-11-23T05:40:20,898 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting f2d4012309ec454498672d6ef81b2a5d, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=388, earliestPutTs=1732340418927 2024-11-23T05:40:20,899 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting cd51f684d57049a1890a595111de750f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732340419104 2024-11-23T05:40:20,899 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 05a704c00f3c4f25b22e186a05582520, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732340419757 2024-11-23T05:40:20,916 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#96 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:20,917 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/0dd46d2b876d4b1fbbe2387fdf5807a7 is 50, key is test_row_0/B:col10/1732340419857/Put/seqid=0 2024-11-23T05:40:20,929 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#97 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:20,929 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/23e8fd9daf0d4d638e283f679c9a4691 is 50, key is test_row_0/A:col10/1732340419857/Put/seqid=0 2024-11-23T05:40:20,946 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741933_1109 (size=13391) 2024-11-23T05:40:20,949 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=28 2024-11-23T05:40:20,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:20,950 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T05:40:20,970 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/0dd46d2b876d4b1fbbe2387fdf5807a7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0dd46d2b876d4b1fbbe2387fdf5807a7 2024-11-23T05:40:20,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741934_1110 (size=13391) 2024-11-23T05:40:20,984 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ddafd66c6876406fb362f58c98ab0286 is 50, key is test_row_0/A:col10/1732340419902/Put/seqid=0 2024-11-23T05:40:20,989 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 0dd46d2b876d4b1fbbe2387fdf5807a7(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:20,989 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:20,990 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340420893; duration=0sec 2024-11-23T05:40:20,990 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:20,990 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:20,990 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 2 store files, 0 compacting, 2 eligible, 16 blocking 2024-11-23T05:40:20,994 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 0 files of size 0 starting at candidate #-1 after considering 0 permutations with 0 in ratio 2024-11-23T05:40:20,994 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(232): Not compacting files because we only have 0 files ready for compaction. Need 3 to initiate. 2024-11-23T05:40:20,995 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. because compaction request was cancelled 2024-11-23T05:40:20,995 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:20,995 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/23e8fd9daf0d4d638e283f679c9a4691 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/23e8fd9daf0d4d638e283f679c9a4691 2024-11-23T05:40:21,004 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 23e8fd9daf0d4d638e283f679c9a4691(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:21,004 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:21,004 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340420893; duration=0sec 2024-11-23T05:40:21,004 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:21,004 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:21,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741935_1111 (size=12301) 2024-11-23T05:40:21,011 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ddafd66c6876406fb362f58c98ab0286 2024-11-23T05:40:21,022 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/492947685dbc4000851499ee2687ae27 is 50, key is test_row_0/B:col10/1732340419902/Put/seqid=0 2024-11-23T05:40:21,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741936_1112 (size=12301) 2024-11-23T05:40:21,033 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/492947685dbc4000851499ee2687ae27 2024-11-23T05:40:21,052 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/75b1cbcd87fc4e75b188a971612e6967 is 50, key is test_row_0/C:col10/1732340419902/Put/seqid=0 2024-11-23T05:40:21,057 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741937_1113 (size=12301) 2024-11-23T05:40:21,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:21,067 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. as already flushing 2024-11-23T05:40:21,086 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340481083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,086 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340481084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 370 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340481084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,090 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340481086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,149 INFO [Thread-154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-154. 2024-11-23T05:40:21,149 DEBUG [Thread-154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,149 INFO [Thread-156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-156. 2024-11-23T05:40:21,149 DEBUG [Thread-156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,150 INFO [Thread-160 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-160. 2024-11-23T05:40:21,150 DEBUG [Thread-160 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,153 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,153 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,153 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,154 DEBUG [Thread-160 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,154 DEBUG [Thread-156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,154 DEBUG [Thread-154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,154 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,156 INFO [Thread-158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-158. 2024-11-23T05:40:21,156 DEBUG [Thread-158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,156 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,156 DEBUG [Thread-158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,156 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,196 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,196 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340481196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340481196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 371 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340481196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340481196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,197 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T05:40:21,407 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,407 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,407 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 372 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52464 deadline: 1732340481407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52458 deadline: 1732340481407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52446 deadline: 1732340481407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,408 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:52430 deadline: 1732340481407, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,408 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:21,409 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:40:21,458 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=450 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/75b1cbcd87fc4e75b188a971612e6967 2024-11-23T05:40:21,468 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/ddafd66c6876406fb362f58c98ab0286 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ddafd66c6876406fb362f58c98ab0286 2024-11-23T05:40:21,472 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ddafd66c6876406fb362f58c98ab0286, entries=150, sequenceid=450, filesize=12.0 K 2024-11-23T05:40:21,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/492947685dbc4000851499ee2687ae27 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/492947685dbc4000851499ee2687ae27 2024-11-23T05:40:21,477 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/492947685dbc4000851499ee2687ae27, entries=150, sequenceid=450, filesize=12.0 K 2024-11-23T05:40:21,479 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/75b1cbcd87fc4e75b188a971612e6967 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/75b1cbcd87fc4e75b188a971612e6967 2024-11-23T05:40:21,483 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/75b1cbcd87fc4e75b188a971612e6967, entries=150, sequenceid=450, filesize=12.0 K 2024-11-23T05:40:21,484 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 96c10c5615cf153b0faba5173b51e570 in 535ms, sequenceid=450, compaction requested=true 2024-11-23T05:40:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:21,484 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=28}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=28 2024-11-23T05:40:21,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=28 2024-11-23T05:40:21,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=28, resume processing ppid=27 2024-11-23T05:40:21,487 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=28, ppid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3140 sec 2024-11-23T05:40:21,488 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=27, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=27, table=TestAcidGuarantees in 1.3210 sec 2024-11-23T05:40:21,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:21,725 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:21,726 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T05:40:21,726 INFO [Thread-143 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-143. 2024-11-23T05:40:21,726 INFO [Thread-149 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-149. 2024-11-23T05:40:21,726 INFO [Thread-151 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-151. 2024-11-23T05:40:21,726 DEBUG [Thread-143 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,726 DEBUG [Thread-149 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,726 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,726 DEBUG [Thread-151 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,726 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,726 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,726 DEBUG [Thread-143 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,726 DEBUG [Thread-149 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,726 DEBUG [Thread-151 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,726 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,727 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,728 INFO [Thread-145 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-145. 2024-11-23T05:40:21,728 DEBUG [Thread-145 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:21,728 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,728 DEBUG [Thread-145 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:21,728 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:21,731 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/1c282655c5c041bcbb521a05cfef0c4a is 50, key is test_row_0/A:col10/1732340421719/Put/seqid=0 2024-11-23T05:40:21,734 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741938_1114 (size=12301) 2024-11-23T05:40:22,135 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/1c282655c5c041bcbb521a05cfef0c4a 2024-11-23T05:40:22,142 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3d7a4bc2743a4422b3df87d8859e06c7 is 50, key is test_row_0/B:col10/1732340421719/Put/seqid=0 2024-11-23T05:40:22,145 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741939_1115 (size=12301) 2024-11-23T05:40:22,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=27 2024-11-23T05:40:22,317 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:22,346 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T05:40:22,546 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3d7a4bc2743a4422b3df87d8859e06c7 2024-11-23T05:40:22,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4de6fe15e65c4ac4b3131753ffa2e666 is 50, key is test_row_0/C:col10/1732340421719/Put/seqid=0 2024-11-23T05:40:22,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741940_1116 (size=12301) 2024-11-23T05:40:22,969 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=470 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4de6fe15e65c4ac4b3131753ffa2e666 2024-11-23T05:40:22,981 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/1c282655c5c041bcbb521a05cfef0c4a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/1c282655c5c041bcbb521a05cfef0c4a 2024-11-23T05:40:22,986 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/1c282655c5c041bcbb521a05cfef0c4a, entries=150, sequenceid=470, filesize=12.0 K 2024-11-23T05:40:22,987 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/3d7a4bc2743a4422b3df87d8859e06c7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3d7a4bc2743a4422b3df87d8859e06c7 2024-11-23T05:40:22,993 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3d7a4bc2743a4422b3df87d8859e06c7, entries=150, sequenceid=470, filesize=12.0 K 2024-11-23T05:40:22,994 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/4de6fe15e65c4ac4b3131753ffa2e666 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4de6fe15e65c4ac4b3131753ffa2e666 2024-11-23T05:40:22,998 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4de6fe15e65c4ac4b3131753ffa2e666, entries=150, sequenceid=470, filesize=12.0 K 2024-11-23T05:40:22,999 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=6.71 KB/6870 for 96c10c5615cf153b0faba5173b51e570 in 1274ms, sequenceid=470, compaction requested=true 2024-11-23T05:40:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:22,999 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:23,000 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:23,000 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96c10c5615cf153b0faba5173b51e570:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:23,000 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:23,001 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:23,001 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37993 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:23,001 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/B is initiating minor compaction (all files) 2024-11-23T05:40:23,001 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/A is initiating minor compaction (all files) 2024-11-23T05:40:23,001 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/A in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:23,001 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/B in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:23,001 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/23e8fd9daf0d4d638e283f679c9a4691, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ddafd66c6876406fb362f58c98ab0286, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/1c282655c5c041bcbb521a05cfef0c4a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=37.1 K 2024-11-23T05:40:23,001 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0dd46d2b876d4b1fbbe2387fdf5807a7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/492947685dbc4000851499ee2687ae27, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3d7a4bc2743a4422b3df87d8859e06c7] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=37.1 K 2024-11-23T05:40:23,001 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 23e8fd9daf0d4d638e283f679c9a4691, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732340419757 2024-11-23T05:40:23,001 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0dd46d2b876d4b1fbbe2387fdf5807a7, keycount=150, bloomtype=ROW, size=13.1 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732340419757 2024-11-23T05:40:23,002 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 492947685dbc4000851499ee2687ae27, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732340419895 2024-11-23T05:40:23,002 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ddafd66c6876406fb362f58c98ab0286, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732340419895 2024-11-23T05:40:23,002 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d7a4bc2743a4422b3df87d8859e06c7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732340421719 2024-11-23T05:40:23,002 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c282655c5c041bcbb521a05cfef0c4a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732340421719 2024-11-23T05:40:23,010 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#B#compaction#105 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:23,010 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#A#compaction#104 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:23,011 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/0cd3e4157183443bbe52918706d4901e is 50, key is test_row_0/B:col10/1732340421719/Put/seqid=0 2024-11-23T05:40:23,011 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/9db3b28d25784594afb12d1fc648ec17 is 50, key is test_row_0/A:col10/1732340421719/Put/seqid=0 2024-11-23T05:40:23,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741941_1117 (size=13493) 2024-11-23T05:40:23,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741942_1118 (size=13493) 2024-11-23T05:40:23,427 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/0cd3e4157183443bbe52918706d4901e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0cd3e4157183443bbe52918706d4901e 2024-11-23T05:40:23,427 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/9db3b28d25784594afb12d1fc648ec17 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9db3b28d25784594afb12d1fc648ec17 2024-11-23T05:40:23,433 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/B of 96c10c5615cf153b0faba5173b51e570 into 0cd3e4157183443bbe52918706d4901e(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:23,433 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/A of 96c10c5615cf153b0faba5173b51e570 into 9db3b28d25784594afb12d1fc648ec17(size=13.2 K), total size for store is 13.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:23,433 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:23,433 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:23,433 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/B, priority=13, startTime=1732340423000; duration=0sec 2024-11-23T05:40:23,433 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/A, priority=13, startTime=1732340422999; duration=0sec 2024-11-23T05:40:23,434 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:23,434 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:B 2024-11-23T05:40:23,434 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:23,434 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:23,434 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:A 2024-11-23T05:40:23,435 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50226 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:23,435 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96c10c5615cf153b0faba5173b51e570/C is initiating minor compaction (all files) 2024-11-23T05:40:23,435 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96c10c5615cf153b0faba5173b51e570/C in TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:23,435 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e53e316a8dc741dda3a9e5d002ea29a9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/b32cbe98e9df414d80c850e2371a6688, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/75b1cbcd87fc4e75b188a971612e6967, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4de6fe15e65c4ac4b3131753ffa2e666] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp, totalSize=49.0 K 2024-11-23T05:40:23,435 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e53e316a8dc741dda3a9e5d002ea29a9, keycount=150, bloomtype=ROW, size=13.0 K, encoding=NONE, compression=NONE, seqNum=411, earliestPutTs=1732340419104 2024-11-23T05:40:23,436 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b32cbe98e9df414d80c850e2371a6688, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=426, earliestPutTs=1732340419757 2024-11-23T05:40:23,436 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 75b1cbcd87fc4e75b188a971612e6967, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=450, earliestPutTs=1732340419895 2024-11-23T05:40:23,436 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4de6fe15e65c4ac4b3131753ffa2e666, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=470, earliestPutTs=1732340421719 2024-11-23T05:40:23,444 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96c10c5615cf153b0faba5173b51e570#C#compaction#106 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:23,445 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/e36e8f38f06144c7bc4cd2041d5cbb45 is 50, key is test_row_0/C:col10/1732340421719/Put/seqid=0 2024-11-23T05:40:23,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741943_1119 (size=13459) 2024-11-23T05:40:23,859 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/e36e8f38f06144c7bc4cd2041d5cbb45 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e36e8f38f06144c7bc4cd2041d5cbb45 2024-11-23T05:40:23,865 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96c10c5615cf153b0faba5173b51e570/C of 96c10c5615cf153b0faba5173b51e570 into e36e8f38f06144c7bc4cd2041d5cbb45(size=13.1 K), total size for store is 13.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:23,865 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96c10c5615cf153b0faba5173b51e570: 2024-11-23T05:40:23,865 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., storeName=96c10c5615cf153b0faba5173b51e570/C, priority=12, startTime=1732340423000; duration=0sec 2024-11-23T05:40:23,865 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:23,865 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96c10c5615cf153b0faba5173b51e570:C 2024-11-23T05:40:29,863 INFO [Thread-147 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-147. 2024-11-23T05:40:29,863 DEBUG [Thread-147 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:29,863 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:29,864 DEBUG [Thread-147 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:29,864 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T05:40:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-23T05:40:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 20 2024-11-23T05:40:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 6 2024-11-23T05:40:29,865 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 309 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 16 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3003 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 2944 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1431 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4289 rows 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1443 2024-11-23T05:40:29,866 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4319 rows 2024-11-23T05:40:29,867 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:40:29,867 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMixedAtomicity(AcidGuaranteesTestBase.java:117) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:40:29,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:29,867 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:29,867 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:29,872 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T05:40:29,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=29, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:29,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T05:40:29,878 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340429878"}]},"ts":"1732340429878"} 2024-11-23T05:40:29,880 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T05:40:29,880 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T05:40:29,882 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=30, ppid=29, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:40:29,887 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, UNASSIGN}] 2024-11-23T05:40:29,888 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, UNASSIGN 2024-11-23T05:40:29,889 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=96c10c5615cf153b0faba5173b51e570, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:29,891 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=31, ppid=30, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, UNASSIGN because future has completed 2024-11-23T05:40:29,892 DEBUG [PEWorker-1 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:40:29,892 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:40:29,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T05:40:30,053 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(122): Close 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:30,053 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:40:30,055 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1722): Closing 96c10c5615cf153b0faba5173b51e570, disabling compactions & flushes 2024-11-23T05:40:30,055 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:30,055 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:30,055 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. after waiting 0 ms 2024-11-23T05:40:30,055 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:30,055 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(2902): Flushing 96c10c5615cf153b0faba5173b51e570 3/3 column families, dataSize=13.42 KB heapSize=35.91 KB 2024-11-23T05:40:30,061 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5aaf3a9581034e0b82fc31ea78a648ee is 50, key is test_row_1/A:col10/1732340429858/Put/seqid=0 2024-11-23T05:40:30,064 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741944_1120 (size=9857) 2024-11-23T05:40:30,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T05:40:30,466 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5aaf3a9581034e0b82fc31ea78a648ee 2024-11-23T05:40:30,480 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/c767e18c44c340f89668698ea49d8185 is 50, key is test_row_1/B:col10/1732340429858/Put/seqid=0 2024-11-23T05:40:30,484 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741945_1121 (size=9857) 2024-11-23T05:40:30,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T05:40:30,885 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/c767e18c44c340f89668698ea49d8185 2024-11-23T05:40:30,892 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ca602c09b6c549398a9dd8cac6d8c7da is 50, key is test_row_1/C:col10/1732340429858/Put/seqid=0 2024-11-23T05:40:30,896 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741946_1122 (size=9857) 2024-11-23T05:40:31,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T05:40:31,297 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=4.47 KB at sequenceid=478 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ca602c09b6c549398a9dd8cac6d8c7da 2024-11-23T05:40:31,302 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/A/5aaf3a9581034e0b82fc31ea78a648ee as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5aaf3a9581034e0b82fc31ea78a648ee 2024-11-23T05:40:31,307 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5aaf3a9581034e0b82fc31ea78a648ee, entries=100, sequenceid=478, filesize=9.6 K 2024-11-23T05:40:31,308 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/B/c767e18c44c340f89668698ea49d8185 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/c767e18c44c340f89668698ea49d8185 2024-11-23T05:40:31,314 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/c767e18c44c340f89668698ea49d8185, entries=100, sequenceid=478, filesize=9.6 K 2024-11-23T05:40:31,315 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/.tmp/C/ca602c09b6c549398a9dd8cac6d8c7da as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ca602c09b6c549398a9dd8cac6d8c7da 2024-11-23T05:40:31,320 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ca602c09b6c549398a9dd8cac6d8c7da, entries=100, sequenceid=478, filesize=9.6 K 2024-11-23T05:40:31,321 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(3140): Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 96c10c5615cf153b0faba5173b51e570 in 1266ms, sequenceid=478, compaction requested=false 2024-11-23T05:40:31,322 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/65835bb3df3344858a0d7e99df4b4ce9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/24d0ea93ccfd40a69fd35103a3769928, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0abe77be8a0a4290821050b5c6aead9c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/667618133cf8438999926917649f917f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5531c63f3fd44d7c81ecc5fb6608e2c9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/06959fa58c6746f48e7b22e4193b5d84, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a22693d019684be992eb1a11cac119cc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/192284d824794389b7d4387ccea074d6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/04688a2b58764aa19e1713e3e6c095a6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/f682de6fa9454164a5f55e921ac23cff, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a0f11811fb2441d191b13d33309a602a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3991dcd755904ee2ba09f9cbc44c23b3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9ee5705175134eaaa860603dccb86460, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/8d8175401aa048bca76df71b230fa5cf, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ba043508be164063acf20b108f89e214, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/e670b30903184a61984dfbd257e55147, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6e0bf3f328d345a79f00995c38729593, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b845bd9a26454c3ca3027bc8515d0c37, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/d57f286270f149bea789fb4dcd04dfe5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ae318036cf40457391f93fe2c490ac3f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/218bbddd3c9147f3a2321cc98742a134, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/bde61d25d5cd4a5ebdca1c1cda695388, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/c854b0361646449bbcebeeee878258fe, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5216f6755a724ba1ba0b5512a72aa3d7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/14df55a31c444fdf8ab0d161ef9fd720, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b61470460128492c827935e4dbeb8fe0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0b9f83961e3048a5bfa139e361813f5d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3feed8686d104d11a2b49b074d09d0e7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/73eda7338efb44dca987d8290b33f630, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/2e8560dfc8be4eebbe52f27f1121819a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5553693e619242939d72234c8e7b55b9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6bc6b2a023844aa78e31738a61481ed6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/23e8fd9daf0d4d638e283f679c9a4691, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ddafd66c6876406fb362f58c98ab0286, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/1c282655c5c041bcbb521a05cfef0c4a] to archive 2024-11-23T05:40:31,325 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:40:31,331 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/65835bb3df3344858a0d7e99df4b4ce9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/65835bb3df3344858a0d7e99df4b4ce9 2024-11-23T05:40:31,332 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/24d0ea93ccfd40a69fd35103a3769928 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/24d0ea93ccfd40a69fd35103a3769928 2024-11-23T05:40:31,333 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0abe77be8a0a4290821050b5c6aead9c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0abe77be8a0a4290821050b5c6aead9c 2024-11-23T05:40:31,334 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/667618133cf8438999926917649f917f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/667618133cf8438999926917649f917f 2024-11-23T05:40:31,335 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5531c63f3fd44d7c81ecc5fb6608e2c9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5531c63f3fd44d7c81ecc5fb6608e2c9 2024-11-23T05:40:31,337 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/06959fa58c6746f48e7b22e4193b5d84 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/06959fa58c6746f48e7b22e4193b5d84 2024-11-23T05:40:31,338 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a22693d019684be992eb1a11cac119cc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a22693d019684be992eb1a11cac119cc 2024-11-23T05:40:31,339 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/192284d824794389b7d4387ccea074d6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/192284d824794389b7d4387ccea074d6 2024-11-23T05:40:31,340 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/04688a2b58764aa19e1713e3e6c095a6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/04688a2b58764aa19e1713e3e6c095a6 2024-11-23T05:40:31,341 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/f682de6fa9454164a5f55e921ac23cff to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/f682de6fa9454164a5f55e921ac23cff 2024-11-23T05:40:31,343 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a0f11811fb2441d191b13d33309a602a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/a0f11811fb2441d191b13d33309a602a 2024-11-23T05:40:31,344 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3991dcd755904ee2ba09f9cbc44c23b3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3991dcd755904ee2ba09f9cbc44c23b3 2024-11-23T05:40:31,345 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9ee5705175134eaaa860603dccb86460 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9ee5705175134eaaa860603dccb86460 2024-11-23T05:40:31,346 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/8d8175401aa048bca76df71b230fa5cf to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/8d8175401aa048bca76df71b230fa5cf 2024-11-23T05:40:31,347 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ba043508be164063acf20b108f89e214 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ba043508be164063acf20b108f89e214 2024-11-23T05:40:31,349 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/e670b30903184a61984dfbd257e55147 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/e670b30903184a61984dfbd257e55147 2024-11-23T05:40:31,350 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6e0bf3f328d345a79f00995c38729593 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6e0bf3f328d345a79f00995c38729593 2024-11-23T05:40:31,352 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b845bd9a26454c3ca3027bc8515d0c37 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b845bd9a26454c3ca3027bc8515d0c37 2024-11-23T05:40:31,353 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/d57f286270f149bea789fb4dcd04dfe5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/d57f286270f149bea789fb4dcd04dfe5 2024-11-23T05:40:31,355 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ae318036cf40457391f93fe2c490ac3f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ae318036cf40457391f93fe2c490ac3f 2024-11-23T05:40:31,356 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/218bbddd3c9147f3a2321cc98742a134 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/218bbddd3c9147f3a2321cc98742a134 2024-11-23T05:40:31,357 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/bde61d25d5cd4a5ebdca1c1cda695388 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/bde61d25d5cd4a5ebdca1c1cda695388 2024-11-23T05:40:31,358 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/c854b0361646449bbcebeeee878258fe to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/c854b0361646449bbcebeeee878258fe 2024-11-23T05:40:31,360 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5216f6755a724ba1ba0b5512a72aa3d7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5216f6755a724ba1ba0b5512a72aa3d7 2024-11-23T05:40:31,361 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/14df55a31c444fdf8ab0d161ef9fd720 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/14df55a31c444fdf8ab0d161ef9fd720 2024-11-23T05:40:31,363 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b61470460128492c827935e4dbeb8fe0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/b61470460128492c827935e4dbeb8fe0 2024-11-23T05:40:31,364 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0b9f83961e3048a5bfa139e361813f5d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/0b9f83961e3048a5bfa139e361813f5d 2024-11-23T05:40:31,365 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3feed8686d104d11a2b49b074d09d0e7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/3feed8686d104d11a2b49b074d09d0e7 2024-11-23T05:40:31,367 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/73eda7338efb44dca987d8290b33f630 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/73eda7338efb44dca987d8290b33f630 2024-11-23T05:40:31,368 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/2e8560dfc8be4eebbe52f27f1121819a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/2e8560dfc8be4eebbe52f27f1121819a 2024-11-23T05:40:31,369 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5553693e619242939d72234c8e7b55b9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5553693e619242939d72234c8e7b55b9 2024-11-23T05:40:31,370 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6bc6b2a023844aa78e31738a61481ed6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/6bc6b2a023844aa78e31738a61481ed6 2024-11-23T05:40:31,371 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/23e8fd9daf0d4d638e283f679c9a4691 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/23e8fd9daf0d4d638e283f679c9a4691 2024-11-23T05:40:31,372 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ddafd66c6876406fb362f58c98ab0286 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/ddafd66c6876406fb362f58c98ab0286 2024-11-23T05:40:31,373 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/1c282655c5c041bcbb521a05cfef0c4a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/1c282655c5c041bcbb521a05cfef0c4a 2024-11-23T05:40:31,387 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b6c143faa8184e83bba42104b0b6add4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b9d198999a740bba209c38607541260, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1bc4aa544a714273b2483e1812a6c4de, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/22ebf8dd87fb4d0e8f9b2052b62dd3cc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/815b3dd454864b1e82411c8255c95b6a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/2ef03c8ef844466cb9e085ceb76b3721, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5bbb545d126c4b5b81ca1c6e1357ee6c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6074f69ac8c140f8bafbd1bab5701168, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1ea8fcc78674478ebc36cc05b3b31d98, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/10eaed887ee245b19f59d949ef616258, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1fae2934803944ef89f7bd4641ac098f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6d8fb43288534ecf825ed140d51a0f88, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/20523c7e3f554bddb8c7aeba56c98615, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5acff5ea78224276a051717a220f4d8e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/291ed6b06fdf4d4e9f7719270d300e37, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3fdb6ea6b4e84c8fb4236743e9b860fa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/7bf0e4b401cb4bebaa444affcffe7638, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6753b46abd2342f39b22c3c9421833b2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/e72b238601b844909451117c64fb58c4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/69fa527ba3a0409991a8ee9d7973c7b9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b1f4b3af92484903bdb0c0c6df454671, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b1d53ccecf045dbb20ef4c6ed20c511, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3c46115628754e3095e6665113d45413, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f1002d58ad5e4b70a5df9dc33c97ed39, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/a6cc857a90a14823b0edf4f51f70ceae, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/d4420e162df54099882773f68d83529b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/645aabcabbc04da69c25aee0b4afd4b9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/98db027fcc2f4aa3be85e432aafa9ff5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f2d4012309ec454498672d6ef81b2a5d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/733c75d6f9b3446988bd9f1e69a4163b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/cd51f684d57049a1890a595111de750f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0dd46d2b876d4b1fbbe2387fdf5807a7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/05a704c00f3c4f25b22e186a05582520, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/492947685dbc4000851499ee2687ae27, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3d7a4bc2743a4422b3df87d8859e06c7] to archive 2024-11-23T05:40:31,388 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:40:31,390 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b6c143faa8184e83bba42104b0b6add4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b6c143faa8184e83bba42104b0b6add4 2024-11-23T05:40:31,391 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b9d198999a740bba209c38607541260 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b9d198999a740bba209c38607541260 2024-11-23T05:40:31,393 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1bc4aa544a714273b2483e1812a6c4de to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1bc4aa544a714273b2483e1812a6c4de 2024-11-23T05:40:31,394 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/22ebf8dd87fb4d0e8f9b2052b62dd3cc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/22ebf8dd87fb4d0e8f9b2052b62dd3cc 2024-11-23T05:40:31,395 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/815b3dd454864b1e82411c8255c95b6a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/815b3dd454864b1e82411c8255c95b6a 2024-11-23T05:40:31,396 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/2ef03c8ef844466cb9e085ceb76b3721 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/2ef03c8ef844466cb9e085ceb76b3721 2024-11-23T05:40:31,397 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5bbb545d126c4b5b81ca1c6e1357ee6c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5bbb545d126c4b5b81ca1c6e1357ee6c 2024-11-23T05:40:31,398 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6074f69ac8c140f8bafbd1bab5701168 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6074f69ac8c140f8bafbd1bab5701168 2024-11-23T05:40:31,399 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1ea8fcc78674478ebc36cc05b3b31d98 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1ea8fcc78674478ebc36cc05b3b31d98 2024-11-23T05:40:31,401 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/10eaed887ee245b19f59d949ef616258 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/10eaed887ee245b19f59d949ef616258 2024-11-23T05:40:31,402 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1fae2934803944ef89f7bd4641ac098f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/1fae2934803944ef89f7bd4641ac098f 2024-11-23T05:40:31,403 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6d8fb43288534ecf825ed140d51a0f88 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6d8fb43288534ecf825ed140d51a0f88 2024-11-23T05:40:31,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/20523c7e3f554bddb8c7aeba56c98615 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/20523c7e3f554bddb8c7aeba56c98615 2024-11-23T05:40:31,404 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5acff5ea78224276a051717a220f4d8e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/5acff5ea78224276a051717a220f4d8e 2024-11-23T05:40:31,405 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/291ed6b06fdf4d4e9f7719270d300e37 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/291ed6b06fdf4d4e9f7719270d300e37 2024-11-23T05:40:31,406 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3fdb6ea6b4e84c8fb4236743e9b860fa to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3fdb6ea6b4e84c8fb4236743e9b860fa 2024-11-23T05:40:31,407 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/7bf0e4b401cb4bebaa444affcffe7638 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/7bf0e4b401cb4bebaa444affcffe7638 2024-11-23T05:40:31,408 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6753b46abd2342f39b22c3c9421833b2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/6753b46abd2342f39b22c3c9421833b2 2024-11-23T05:40:31,409 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/e72b238601b844909451117c64fb58c4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/e72b238601b844909451117c64fb58c4 2024-11-23T05:40:31,411 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/69fa527ba3a0409991a8ee9d7973c7b9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/69fa527ba3a0409991a8ee9d7973c7b9 2024-11-23T05:40:31,412 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b1f4b3af92484903bdb0c0c6df454671 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/b1f4b3af92484903bdb0c0c6df454671 2024-11-23T05:40:31,413 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b1d53ccecf045dbb20ef4c6ed20c511 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/4b1d53ccecf045dbb20ef4c6ed20c511 2024-11-23T05:40:31,414 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3c46115628754e3095e6665113d45413 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3c46115628754e3095e6665113d45413 2024-11-23T05:40:31,415 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f1002d58ad5e4b70a5df9dc33c97ed39 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f1002d58ad5e4b70a5df9dc33c97ed39 2024-11-23T05:40:31,416 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/a6cc857a90a14823b0edf4f51f70ceae to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/a6cc857a90a14823b0edf4f51f70ceae 2024-11-23T05:40:31,417 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/d4420e162df54099882773f68d83529b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/d4420e162df54099882773f68d83529b 2024-11-23T05:40:31,418 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/645aabcabbc04da69c25aee0b4afd4b9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/645aabcabbc04da69c25aee0b4afd4b9 2024-11-23T05:40:31,419 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/98db027fcc2f4aa3be85e432aafa9ff5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/98db027fcc2f4aa3be85e432aafa9ff5 2024-11-23T05:40:31,420 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f2d4012309ec454498672d6ef81b2a5d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/f2d4012309ec454498672d6ef81b2a5d 2024-11-23T05:40:31,421 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/733c75d6f9b3446988bd9f1e69a4163b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/733c75d6f9b3446988bd9f1e69a4163b 2024-11-23T05:40:31,422 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/cd51f684d57049a1890a595111de750f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/cd51f684d57049a1890a595111de750f 2024-11-23T05:40:31,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0dd46d2b876d4b1fbbe2387fdf5807a7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0dd46d2b876d4b1fbbe2387fdf5807a7 2024-11-23T05:40:31,423 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/05a704c00f3c4f25b22e186a05582520 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/05a704c00f3c4f25b22e186a05582520 2024-11-23T05:40:31,424 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/492947685dbc4000851499ee2687ae27 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/492947685dbc4000851499ee2687ae27 2024-11-23T05:40:31,425 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3d7a4bc2743a4422b3df87d8859e06c7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/3d7a4bc2743a4422b3df87d8859e06c7 2024-11-23T05:40:31,427 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/628da39df3c64ffda6fdf75a9324cc70, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d25d22460b0a4725a5190a64ef28d490, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9737425bfa2041ffa80fed98d738ae31, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1d4f981d18d3442dbcd819de8f5e949e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/aa05e380d7da4c9bbddeb0b263e326b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1b605a49168c45ab92cb3f0e4ff2f308, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d3ae6a9abb404725a6342bb2321eca99, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f339465b0dfd433b8697251c2512d1d5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d4d4340199834354a8effcce81c3d0aa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/48ca20cbd601450abf0e30b7dd04c331, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4f61b5b6e16c4999b427f94b48a2567a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4698848bffb945349252373d34942a73, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/853dc829cc214e4485a936423483f6be, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ebfa8a6183bf46478a8b9d3ff7b483fd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/cacaeb92cace4222ae8688e88c677cc8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1f3e0abef64b48048ee9199ef196362d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/6260fa23691d4064a06ffe295d3fee28, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/2808b2c0e1574eccaf4c387f46dd5394, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4cb85109dad44fb1bb58adb48b2394e7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/596b8060689b43db8f8b83afede4a194, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/fece152f9d504ca2ab1cc7f46f00126a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ae31dca65ed94f149c32390c28e23e4e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/dfc02ba24e4042cca05586e5f13f4baf, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/8b4f9c7632394ec4942095a86526133e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4eb6f89ff8e34e1786d7e884bc0dc33b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f423eba513f0418dae4698189a32ad0b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/0de548a4ceb1411088ed9021453494bc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9b3f591edb784c919cf24e57ef58bef3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/83bddaf53206453d8afa5e7434673622, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e53e316a8dc741dda3a9e5d002ea29a9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/a4919a238c4f425e931cfc19242bf8e4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/b32cbe98e9df414d80c850e2371a6688, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/75b1cbcd87fc4e75b188a971612e6967, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4de6fe15e65c4ac4b3131753ffa2e666] to archive 2024-11-23T05:40:31,428 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:40:31,429 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/628da39df3c64ffda6fdf75a9324cc70 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/628da39df3c64ffda6fdf75a9324cc70 2024-11-23T05:40:31,431 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d25d22460b0a4725a5190a64ef28d490 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d25d22460b0a4725a5190a64ef28d490 2024-11-23T05:40:31,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9737425bfa2041ffa80fed98d738ae31 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9737425bfa2041ffa80fed98d738ae31 2024-11-23T05:40:31,432 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1d4f981d18d3442dbcd819de8f5e949e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1d4f981d18d3442dbcd819de8f5e949e 2024-11-23T05:40:31,433 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/aa05e380d7da4c9bbddeb0b263e326b7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/aa05e380d7da4c9bbddeb0b263e326b7 2024-11-23T05:40:31,435 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1b605a49168c45ab92cb3f0e4ff2f308 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1b605a49168c45ab92cb3f0e4ff2f308 2024-11-23T05:40:31,436 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d3ae6a9abb404725a6342bb2321eca99 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d3ae6a9abb404725a6342bb2321eca99 2024-11-23T05:40:31,437 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f339465b0dfd433b8697251c2512d1d5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f339465b0dfd433b8697251c2512d1d5 2024-11-23T05:40:31,439 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d4d4340199834354a8effcce81c3d0aa to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/d4d4340199834354a8effcce81c3d0aa 2024-11-23T05:40:31,440 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/48ca20cbd601450abf0e30b7dd04c331 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/48ca20cbd601450abf0e30b7dd04c331 2024-11-23T05:40:31,442 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4f61b5b6e16c4999b427f94b48a2567a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4f61b5b6e16c4999b427f94b48a2567a 2024-11-23T05:40:31,443 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4698848bffb945349252373d34942a73 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4698848bffb945349252373d34942a73 2024-11-23T05:40:31,444 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/853dc829cc214e4485a936423483f6be to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/853dc829cc214e4485a936423483f6be 2024-11-23T05:40:31,446 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ebfa8a6183bf46478a8b9d3ff7b483fd to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ebfa8a6183bf46478a8b9d3ff7b483fd 2024-11-23T05:40:31,447 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/cacaeb92cace4222ae8688e88c677cc8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/cacaeb92cace4222ae8688e88c677cc8 2024-11-23T05:40:31,448 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1f3e0abef64b48048ee9199ef196362d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/1f3e0abef64b48048ee9199ef196362d 2024-11-23T05:40:31,450 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/6260fa23691d4064a06ffe295d3fee28 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/6260fa23691d4064a06ffe295d3fee28 2024-11-23T05:40:31,451 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/2808b2c0e1574eccaf4c387f46dd5394 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/2808b2c0e1574eccaf4c387f46dd5394 2024-11-23T05:40:31,453 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4cb85109dad44fb1bb58adb48b2394e7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4cb85109dad44fb1bb58adb48b2394e7 2024-11-23T05:40:31,454 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/596b8060689b43db8f8b83afede4a194 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/596b8060689b43db8f8b83afede4a194 2024-11-23T05:40:31,456 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/fece152f9d504ca2ab1cc7f46f00126a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/fece152f9d504ca2ab1cc7f46f00126a 2024-11-23T05:40:31,458 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ae31dca65ed94f149c32390c28e23e4e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ae31dca65ed94f149c32390c28e23e4e 2024-11-23T05:40:31,459 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/dfc02ba24e4042cca05586e5f13f4baf to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/dfc02ba24e4042cca05586e5f13f4baf 2024-11-23T05:40:31,461 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/8b4f9c7632394ec4942095a86526133e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/8b4f9c7632394ec4942095a86526133e 2024-11-23T05:40:31,463 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4eb6f89ff8e34e1786d7e884bc0dc33b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4eb6f89ff8e34e1786d7e884bc0dc33b 2024-11-23T05:40:31,464 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f423eba513f0418dae4698189a32ad0b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/f423eba513f0418dae4698189a32ad0b 2024-11-23T05:40:31,465 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/0de548a4ceb1411088ed9021453494bc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/0de548a4ceb1411088ed9021453494bc 2024-11-23T05:40:31,467 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9b3f591edb784c919cf24e57ef58bef3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/9b3f591edb784c919cf24e57ef58bef3 2024-11-23T05:40:31,469 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/83bddaf53206453d8afa5e7434673622 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/83bddaf53206453d8afa5e7434673622 2024-11-23T05:40:31,470 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e53e316a8dc741dda3a9e5d002ea29a9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e53e316a8dc741dda3a9e5d002ea29a9 2024-11-23T05:40:31,471 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/a4919a238c4f425e931cfc19242bf8e4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/a4919a238c4f425e931cfc19242bf8e4 2024-11-23T05:40:31,473 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/b32cbe98e9df414d80c850e2371a6688 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/b32cbe98e9df414d80c850e2371a6688 2024-11-23T05:40:31,474 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/75b1cbcd87fc4e75b188a971612e6967 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/75b1cbcd87fc4e75b188a971612e6967 2024-11-23T05:40:31,475 DEBUG [StoreCloser-TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4de6fe15e65c4ac4b3131753ffa2e666 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/4de6fe15e65c4ac4b3131753ffa2e666 2024-11-23T05:40:31,485 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/recovered.edits/481.seqid, newMaxSeqId=481, maxSeqId=1 2024-11-23T05:40:31,486 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. 2024-11-23T05:40:31,487 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] regionserver.HRegion(1676): Region close journal for 96c10c5615cf153b0faba5173b51e570: Waiting for close lock at 1732340430055Running coprocessor pre-close hooks at 1732340430055Disabling compacts and flushes for region at 1732340430055Disabling writes for close at 1732340430055Obtaining lock to block concurrent updates at 1732340430055Preparing flush snapshotting stores in 96c10c5615cf153b0faba5173b51e570 at 1732340430055Finished memstore snapshotting TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570., syncing WAL and waiting on mvcc, flushsize=dataSize=13740, getHeapSize=36720, getOffHeapSize=0, getCellsCount=300 at 1732340430056 (+1 ms)Flushing stores of TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570. at 1732340430056Flushing 96c10c5615cf153b0faba5173b51e570/A: creating writer at 1732340430056Flushing 96c10c5615cf153b0faba5173b51e570/A: appending metadata at 1732340430060 (+4 ms)Flushing 96c10c5615cf153b0faba5173b51e570/A: closing flushed file at 1732340430060Flushing 96c10c5615cf153b0faba5173b51e570/B: creating writer at 1732340430476 (+416 ms)Flushing 96c10c5615cf153b0faba5173b51e570/B: appending metadata at 1732340430479 (+3 ms)Flushing 96c10c5615cf153b0faba5173b51e570/B: closing flushed file at 1732340430479Flushing 96c10c5615cf153b0faba5173b51e570/C: creating writer at 1732340430889 (+410 ms)Flushing 96c10c5615cf153b0faba5173b51e570/C: appending metadata at 1732340430892 (+3 ms)Flushing 96c10c5615cf153b0faba5173b51e570/C: closing flushed file at 1732340430892Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@369bbff5: reopening flushed file at 1732340431301 (+409 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@71f1ccd8: reopening flushed file at 1732340431307 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@ff91b31: reopening flushed file at 1732340431314 (+7 ms)Finished flush of dataSize ~13.42 KB/13740, heapSize ~35.86 KB/36720, currentSize=0 B/0 for 96c10c5615cf153b0faba5173b51e570 in 1266ms, sequenceid=478, compaction requested=false at 1732340431321 (+7 ms)Writing region close event to WAL at 1732340431477 (+156 ms)Running coprocessor post-close hooks at 1732340431486 (+9 ms)Closed at 1732340431486 2024-11-23T05:40:31,489 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=32}] handler.UnassignRegionHandler(157): Closed 96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:31,490 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=31 updating hbase:meta row=96c10c5615cf153b0faba5173b51e570, regionState=CLOSED 2024-11-23T05:40:31,492 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=32, ppid=31, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:40:31,495 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=32, resume processing ppid=31 2024-11-23T05:40:31,496 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=32, ppid=31, state=SUCCESS, hasLock=false; CloseRegionProcedure 96c10c5615cf153b0faba5173b51e570, server=9abfc7c2569e,45719,1732340394120 in 1.6010 sec 2024-11-23T05:40:31,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=31, resume processing ppid=30 2024-11-23T05:40:31,497 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=31, ppid=30, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96c10c5615cf153b0faba5173b51e570, UNASSIGN in 1.6100 sec 2024-11-23T05:40:31,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=30, resume processing ppid=29 2024-11-23T05:40:31,500 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=30, ppid=29, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.6160 sec 2024-11-23T05:40:31,502 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340431501"}]},"ts":"1732340431501"} 2024-11-23T05:40:31,504 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T05:40:31,504 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T05:40:31,507 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=29, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.6320 sec 2024-11-23T05:40:32,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=29 2024-11-23T05:40:32,026 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:32,029 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T05:40:32,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:32,034 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=33, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:32,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-23T05:40:32,036 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=33, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:32,040 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:32,044 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/recovered.edits] 2024-11-23T05:40:32,047 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5aaf3a9581034e0b82fc31ea78a648ee to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/5aaf3a9581034e0b82fc31ea78a648ee 2024-11-23T05:40:32,048 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9db3b28d25784594afb12d1fc648ec17 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/A/9db3b28d25784594afb12d1fc648ec17 2024-11-23T05:40:32,051 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0cd3e4157183443bbe52918706d4901e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/0cd3e4157183443bbe52918706d4901e 2024-11-23T05:40:32,052 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/c767e18c44c340f89668698ea49d8185 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/B/c767e18c44c340f89668698ea49d8185 2024-11-23T05:40:32,054 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ca602c09b6c549398a9dd8cac6d8c7da to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/ca602c09b6c549398a9dd8cac6d8c7da 2024-11-23T05:40:32,055 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e36e8f38f06144c7bc4cd2041d5cbb45 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/C/e36e8f38f06144c7bc4cd2041d5cbb45 2024-11-23T05:40:32,058 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/recovered.edits/481.seqid to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570/recovered.edits/481.seqid 2024-11-23T05:40:32,059 DEBUG [HFileArchiver-1 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96c10c5615cf153b0faba5173b51e570 2024-11-23T05:40:32,059 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T05:40:32,065 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=33, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:32,070 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] util.ReflectedFunctionCache(97): Populated cache for org.apache.hadoop.hbase.filter.KeyOnlyFilter in 0ms 2024-11-23T05:40:32,075 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T05:40:32,102 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T05:40:32,103 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=33, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:32,103 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T05:40:32,104 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732340432104"}]},"ts":"9223372036854775807"} 2024-11-23T05:40:32,108 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T05:40:32,108 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 96c10c5615cf153b0faba5173b51e570, NAME => 'TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T05:40:32,108 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T05:40:32,108 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732340432108"}]},"ts":"9223372036854775807"} 2024-11-23T05:40:32,111 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T05:40:32,112 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=33, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:32,113 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=33, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 83 msec 2024-11-23T05:40:32,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=33 2024-11-23T05:40:32,147 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T05:40:32,147 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:32,158 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMixedAtomicity Thread=237 (was 211) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-10 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935771990_22 at /127.0.0.1:56858 [Waiting for operation #13] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS:0;9abfc7c2569e:45719-shortCompactions-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:535) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-9 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-12 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-6 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-8 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-5 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Timer for 'HBase' metrics system java.base@17.0.11/java.lang.Object.wait(Native Method) java.base@17.0.11/java.util.TimerThread.mainLoop(Timer.java:563) java.base@17.0.11/java.util.TimerThread.run(Timer.java:516) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-16 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-7 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-4 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:62) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:883) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-14 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-15 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-11 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RSProcedureDispatcher-pool-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RPCClient-NioEventLoopGroup-4-13 java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:146) app//org.apache.hbase.thirdparty.io.netty.channel.nio.SelectedSelectionKeySetSelector.select(SelectedSelectionKeySetSelector.java:68) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.select(NioEventLoop.java:879) app//org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:526) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) app//org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) app//org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 441) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=497 (was 313) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8426 (was 9389) 2024-11-23T05:40:32,170 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=237, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=497, ProcessCount=11, AvailableMemoryMB=8426 2024-11-23T05:40:32,171 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:40:32,172 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:40:32,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:32,176 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:40:32,176 DEBUG [PEWorker-5 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:32,176 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 34 2024-11-23T05:40:32,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-23T05:40:32,178 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:40:32,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741947_1123 (size=959) 2024-11-23T05:40:32,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-23T05:40:32,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-23T05:40:32,590 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:40:32,600 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741948_1124 (size=53) 2024-11-23T05:40:32,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-23T05:40:33,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:40:33,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 6fed8128b8c4dd3776e5696d683dbd53, disabling compactions & flushes 2024-11-23T05:40:33,001 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:33,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:33,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. after waiting 0 ms 2024-11-23T05:40:33,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:33,001 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:33,001 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 6fed8128b8c4dd3776e5696d683dbd53: Waiting for close lock at 1732340433001Disabling compacts and flushes for region at 1732340433001Disabling writes for close at 1732340433001Writing region close event to WAL at 1732340433001Closed at 1732340433001 2024-11-23T05:40:33,003 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:40:33,004 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732340433003"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732340433003"}]},"ts":"1732340433003"} 2024-11-23T05:40:33,006 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:40:33,007 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:40:33,008 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340433007"}]},"ts":"1732340433007"} 2024-11-23T05:40:33,010 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T05:40:33,010 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, ASSIGN}] 2024-11-23T05:40:33,012 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, ASSIGN 2024-11-23T05:40:33,013 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, ASSIGN; state=OFFLINE, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=false 2024-11-23T05:40:33,164 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:33,167 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=35, ppid=34, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, ASSIGN because future has completed 2024-11-23T05:40:33,167 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:40:33,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-23T05:40:33,324 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:33,324 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:40:33,324 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,324 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:40:33,324 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7794): checking encryption for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,325 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(7797): checking classloading for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,328 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,331 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fed8128b8c4dd3776e5696d683dbd53 columnFamilyName A 2024-11-23T05:40:33,331 DEBUG [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:33,332 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(327): Store=6fed8128b8c4dd3776e5696d683dbd53/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:33,332 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,334 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fed8128b8c4dd3776e5696d683dbd53 columnFamilyName B 2024-11-23T05:40:33,334 DEBUG [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:33,335 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(327): Store=6fed8128b8c4dd3776e5696d683dbd53/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:33,335 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,338 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fed8128b8c4dd3776e5696d683dbd53 columnFamilyName C 2024-11-23T05:40:33,338 DEBUG [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:33,338 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(327): Store=6fed8128b8c4dd3776e5696d683dbd53/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:33,339 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1038): replaying wal for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,340 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,340 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,341 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1048): stopping wal replay for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,341 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1060): Cleaning up temporary data for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,342 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:40:33,343 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1093): writing seq id for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,347 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:40:33,348 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1114): Opened 6fed8128b8c4dd3776e5696d683dbd53; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=72949508, jitterRate=0.08703237771987915}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:40:33,348 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:33,349 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegion(1006): Region open journal for 6fed8128b8c4dd3776e5696d683dbd53: Running coprocessor pre-open hook at 1732340433325Writing region info on filesystem at 1732340433325Initializing all the Stores at 1732340433327 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340433327Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340433328 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340433328Cleaning up temporary data from old regions at 1732340433341 (+13 ms)Running coprocessor post-open hooks at 1732340433348 (+7 ms)Region opened successfully at 1732340433348 2024-11-23T05:40:33,350 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., pid=36, masterSystemTime=1732340433320 2024-11-23T05:40:33,352 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:33,352 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=36}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:33,352 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=35 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:33,354 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=36, ppid=35, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:40:33,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=36, resume processing ppid=35 2024-11-23T05:40:33,363 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=36, ppid=35, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 in 193 msec 2024-11-23T05:40:33,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=35, resume processing ppid=34 2024-11-23T05:40:33,366 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=35, ppid=34, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, ASSIGN in 353 msec 2024-11-23T05:40:33,368 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:40:33,368 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340433368"}]},"ts":"1732340433368"} 2024-11-23T05:40:33,371 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T05:40:33,372 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=34, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:40:33,375 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=34, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.2000 sec 2024-11-23T05:40:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=34 2024-11-23T05:40:34,336 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:34,337 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@594c9324, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:34,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:34,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:34,339 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19818525, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:34,340 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:34,341 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:34,342 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:34,343 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a13e1f2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:34,343 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:34,344 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:34,345 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:34,347 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43038, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:34,349 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:40:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,40189,1732340393249 2024-11-23T05:40:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@6479eaa0 2024-11-23T05:40:34,350 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:40:34,351 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51322, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:40:34,357 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:40:34,357 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:40:34,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=37, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T05:40:34,377 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741949_1125 (size=995) 2024-11-23T05:40:34,781 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T05:40:34,781 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T05:40:34,788 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=38, ppid=37, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:40:34,796 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, REOPEN/MOVE}] 2024-11-23T05:40:34,797 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, REOPEN/MOVE 2024-11-23T05:40:34,798 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:34,800 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, REOPEN/MOVE because future has completed 2024-11-23T05:40:34,801 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:40:34,801 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:40:34,954 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(122): Close 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:34,954 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:40:34,954 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1722): Closing 6fed8128b8c4dd3776e5696d683dbd53, disabling compactions & flushes 2024-11-23T05:40:34,954 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:34,954 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:34,954 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. after waiting 0 ms 2024-11-23T05:40:34,954 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:34,959 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T05:40:34,960 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:34,960 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegion(1676): Region close journal for 6fed8128b8c4dd3776e5696d683dbd53: Waiting for close lock at 1732340434954Running coprocessor pre-close hooks at 1732340434954Disabling compacts and flushes for region at 1732340434954Disabling writes for close at 1732340434954Writing region close event to WAL at 1732340434955 (+1 ms)Running coprocessor post-close hooks at 1732340434960 (+5 ms)Closed at 1732340434960 2024-11-23T05:40:34,961 WARN [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] regionserver.HRegionServer(3299): Not adding moved region record: 6fed8128b8c4dd3776e5696d683dbd53 to self. 2024-11-23T05:40:34,963 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=40}] handler.UnassignRegionHandler(157): Closed 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:34,963 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=CLOSED 2024-11-23T05:40:34,965 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=40, ppid=39, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:40:34,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=40, resume processing ppid=39 2024-11-23T05:40:34,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=40, ppid=39, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 in 166 msec 2024-11-23T05:40:34,970 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, REOPEN/MOVE; state=CLOSED, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=true 2024-11-23T05:40:35,121 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,129 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=39, ppid=38, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, REOPEN/MOVE because future has completed 2024-11-23T05:40:35,129 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:40:35,285 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,285 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7752): Opening region: {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:40:35,286 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,286 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:40:35,286 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7794): checking encryption for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,286 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(7797): checking classloading for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,288 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,294 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fed8128b8c4dd3776e5696d683dbd53 columnFamilyName A 2024-11-23T05:40:35,296 DEBUG [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:35,297 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(327): Store=6fed8128b8c4dd3776e5696d683dbd53/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:35,298 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,299 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fed8128b8c4dd3776e5696d683dbd53 columnFamilyName B 2024-11-23T05:40:35,299 DEBUG [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:35,299 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(327): Store=6fed8128b8c4dd3776e5696d683dbd53/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:35,299 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,300 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 6fed8128b8c4dd3776e5696d683dbd53 columnFamilyName C 2024-11-23T05:40:35,300 DEBUG [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:35,301 INFO [StoreOpener-6fed8128b8c4dd3776e5696d683dbd53-1 {}] regionserver.HStore(327): Store=6fed8128b8c4dd3776e5696d683dbd53/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:40:35,301 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1038): replaying wal for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,301 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,302 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,303 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1048): stopping wal replay for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,303 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1060): Cleaning up temporary data for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,303 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:40:35,304 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1093): writing seq id for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,305 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1114): Opened 6fed8128b8c4dd3776e5696d683dbd53; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=69088538, jitterRate=0.029499441385269165}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:40:35,305 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,307 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegion(1006): Region open journal for 6fed8128b8c4dd3776e5696d683dbd53: Running coprocessor pre-open hook at 1732340435286Writing region info on filesystem at 1732340435286Initializing all the Stores at 1732340435287 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340435287Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340435287Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340435287Cleaning up temporary data from old regions at 1732340435303 (+16 ms)Running coprocessor post-open hooks at 1732340435305 (+2 ms)Region opened successfully at 1732340435307 (+2 ms) 2024-11-23T05:40:35,308 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., pid=41, masterSystemTime=1732340435282 2024-11-23T05:40:35,310 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,310 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=41}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,310 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=39 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=OPEN, openSeqNum=5, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,312 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=41, ppid=39, state=RUNNABLE, hasLock=false; OpenRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:40:35,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=41, resume processing ppid=39 2024-11-23T05:40:35,315 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=41, ppid=39, state=SUCCESS, hasLock=false; OpenRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 in 184 msec 2024-11-23T05:40:35,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=39, resume processing ppid=38 2024-11-23T05:40:35,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=39, ppid=38, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, REOPEN/MOVE in 519 msec 2024-11-23T05:40:35,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=38, resume processing ppid=37 2024-11-23T05:40:35,321 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=38, ppid=37, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 532 msec 2024-11-23T05:40:35,324 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=37, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 962 msec 2024-11-23T05:40:35,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=37 2024-11-23T05:40:35,326 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:35,327 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fcb20d1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,327 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,327 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,328 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@68d1f829, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,328 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,329 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,330 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51346, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,331 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@389942aa, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,331 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2e853879, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,331 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,332 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,333 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,333 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f2ecc06, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,334 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,335 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,336 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1e7a6f22, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,338 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5a00ea56, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,338 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,338 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,339 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@180c9a8c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,340 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,341 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51380, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,342 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b2c23e2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,344 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@636d090d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,344 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,344 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,345 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@329323bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,345 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,346 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,347 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51400, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,348 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@405b4eeb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,351 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5e0133ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,351 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,351 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,352 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,352 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@688e55a3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,353 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,354 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51410, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,355 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6641e70c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,356 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@bc5db83, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,356 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,356 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,357 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,357 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3a4f716b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,358 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,360 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51418, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,361 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@139083ef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,362 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4870d163, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,362 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,362 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,363 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,363 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523bf26b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,364 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,365 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51440, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,367 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65d0ad0a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,371 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@53e9634b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,371 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,371 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,372 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6a1f9e53, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,373 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,374 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,375 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51458, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,375 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@a52b838, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,379 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54f7e914, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,379 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:40:35,379 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:40:35,380 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:40:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:40:35,380 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:40:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@569374a5, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:40:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:40:35,381 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:35,382 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:51478, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:40:35,383 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3035364d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:40:35,384 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:35,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees 2024-11-23T05:40:35,387 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:35,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T05:40:35,388 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=42, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:35,388 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=43, ppid=42, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:35,391 DEBUG [Thread-639 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,391 DEBUG [Thread-637 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,392 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,393 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43052, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,394 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,399 DEBUG [Thread-641 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,399 DEBUG [Thread-644 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,400 DEBUG [Thread-633 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,401 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,402 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,403 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,404 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,405 DEBUG [Thread-635 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,405 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43082, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,405 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,408 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,409 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,410 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,412 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,417 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:35,417 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:40:35,427 DEBUG [Thread-648 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,428 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,429 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43102, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,431 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,435 DEBUG [Thread-646 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,436 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,436 DEBUG [Thread-646 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,438 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43110, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,446 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340495445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,452 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,453 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340495446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,454 DEBUG [Thread-650 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:40:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,454 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:40:35,455 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:40:35,457 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:43114, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:40:35,457 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340495448, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,459 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:40:35,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340495451, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,460 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340495456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,460 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T05:40:35,499 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d9114b4093a840c39e9b31388bbfe79d_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340435415/Put/seqid=0 2024-11-23T05:40:35,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T05:40:35,541 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741950_1126 (size=12154) 2024-11-23T05:40:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,542 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:35,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,549 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d9114b4093a840c39e9b31388bbfe79d_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d9114b4093a840c39e9b31388bbfe79d_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:35,550 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/ef6a8452e33d469bb9d759463f60477f, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:35,559 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/ef6a8452e33d469bb9d759463f60477f is 175, key is test_row_0/A:col10/1732340435415/Put/seqid=0 2024-11-23T05:40:35,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340495557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,565 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340495557, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,566 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340495566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,571 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340495566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340495567, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,573 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,601 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741951_1127 (size=30955) 2024-11-23T05:40:35,695 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T05:40:35,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:35,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T05:40:35,783 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340495777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,784 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340495777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,784 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,785 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340495777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340495777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340495777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:35,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:35,850 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T05:40:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:35,850 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,850 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:35,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:36,004 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T05:40:36,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:36,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:36,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:36,005 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:36,005 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:36,006 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=17, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/ef6a8452e33d469bb9d759463f60477f 2024-11-23T05:40:36,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:36,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T05:40:36,047 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/72adc6dbbdeb42c8b9dafca84cd3c167 is 50, key is test_row_0/B:col10/1732340435415/Put/seqid=0 2024-11-23T05:40:36,076 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741952_1128 (size=12001) 2024-11-23T05:40:36,078 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/72adc6dbbdeb42c8b9dafca84cd3c167 2024-11-23T05:40:36,102 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,103 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340496097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,103 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,104 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340496097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,105 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,106 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340496097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340496097, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,106 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,106 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340496098, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,107 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,131 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/da01845e9cde4866bb3ef2373990191d is 50, key is test_row_0/C:col10/1732340435415/Put/seqid=0 2024-11-23T05:40:36,159 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T05:40:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:36,159 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] handler.RSProcedureHandler(58): pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:36,159 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=43 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:36,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=43 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:36,178 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741953_1129 (size=12001) 2024-11-23T05:40:36,180 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=17 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/da01845e9cde4866bb3ef2373990191d 2024-11-23T05:40:36,191 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/ef6a8452e33d469bb9d759463f60477f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/ef6a8452e33d469bb9d759463f60477f 2024-11-23T05:40:36,205 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/ef6a8452e33d469bb9d759463f60477f, entries=150, sequenceid=17, filesize=30.2 K 2024-11-23T05:40:36,214 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/72adc6dbbdeb42c8b9dafca84cd3c167 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/72adc6dbbdeb42c8b9dafca84cd3c167 2024-11-23T05:40:36,227 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/72adc6dbbdeb42c8b9dafca84cd3c167, entries=150, sequenceid=17, filesize=11.7 K 2024-11-23T05:40:36,228 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/da01845e9cde4866bb3ef2373990191d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/da01845e9cde4866bb3ef2373990191d 2024-11-23T05:40:36,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/da01845e9cde4866bb3ef2373990191d, entries=150, sequenceid=17, filesize=11.7 K 2024-11-23T05:40:36,237 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for 6fed8128b8c4dd3776e5696d683dbd53 in 820ms, sequenceid=17, compaction requested=false 2024-11-23T05:40:36,237 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T05:40:36,238 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:36,314 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=43 2024-11-23T05:40:36,314 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:36,315 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:40:36,343 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eeb7f98333704d3a8dc86e6104514b99_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340435444/Put/seqid=0 2024-11-23T05:40:36,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741954_1130 (size=12154) 2024-11-23T05:40:36,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:36,393 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eeb7f98333704d3a8dc86e6104514b99_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eeb7f98333704d3a8dc86e6104514b99_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:36,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a8ede2a63a19443db15e397b8e863b22, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:36,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a8ede2a63a19443db15e397b8e863b22 is 175, key is test_row_0/A:col10/1732340435444/Put/seqid=0 2024-11-23T05:40:36,418 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741955_1131 (size=30955) 2024-11-23T05:40:36,419 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=41, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a8ede2a63a19443db15e397b8e863b22 2024-11-23T05:40:36,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/0e1796d13d90473598561ca5ba1d8460 is 50, key is test_row_0/B:col10/1732340435444/Put/seqid=0 2024-11-23T05:40:36,448 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741956_1132 (size=12001) 2024-11-23T05:40:36,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T05:40:36,621 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:36,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:36,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340496628, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,641 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,642 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340496637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340496636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,643 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,643 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340496638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,644 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,645 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340496639, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,646 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,748 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340496746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,749 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340496747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,750 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,750 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340496747, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,751 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,757 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340496756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,758 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340496756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:36,758 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:36,759 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:36,849 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/0e1796d13d90473598561ca5ba1d8460 2024-11-23T05:40:36,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/514e815c87e54e66bfa3e7f30d53c008 is 50, key is test_row_0/C:col10/1732340435444/Put/seqid=0 2024-11-23T05:40:36,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741957_1133 (size=12001) 2024-11-23T05:40:36,875 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=41 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/514e815c87e54e66bfa3e7f30d53c008 2024-11-23T05:40:36,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a8ede2a63a19443db15e397b8e863b22 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a8ede2a63a19443db15e397b8e863b22 2024-11-23T05:40:36,894 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a8ede2a63a19443db15e397b8e863b22, entries=150, sequenceid=41, filesize=30.2 K 2024-11-23T05:40:36,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/0e1796d13d90473598561ca5ba1d8460 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0e1796d13d90473598561ca5ba1d8460 2024-11-23T05:40:36,910 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0e1796d13d90473598561ca5ba1d8460, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T05:40:36,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/514e815c87e54e66bfa3e7f30d53c008 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/514e815c87e54e66bfa3e7f30d53c008 2024-11-23T05:40:36,917 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/514e815c87e54e66bfa3e7f30d53c008, entries=150, sequenceid=41, filesize=11.7 K 2024-11-23T05:40:36,926 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6fed8128b8c4dd3776e5696d683dbd53 in 611ms, sequenceid=41, compaction requested=false 2024-11-23T05:40:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:36,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=43}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=43 2024-11-23T05:40:36,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=43 2024-11-23T05:40:36,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=43, resume processing ppid=42 2024-11-23T05:40:36,933 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=43, ppid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5420 sec 2024-11-23T05:40:36,935 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=42, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=42, table=TestAcidGuarantees in 1.5480 sec 2024-11-23T05:40:36,958 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:36,959 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:40:36,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:36,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e383108738ff4d6497a0cb6a177ad9f2_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340436957/Put/seqid=0 2024-11-23T05:40:37,017 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340497007, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,018 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340497012, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,018 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741958_1134 (size=14594) 2024-11-23T05:40:37,025 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:37,025 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340497015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,026 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,027 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,029 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:40:37,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340497017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340497017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,037 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e383108738ff4d6497a0cb6a177ad9f2_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e383108738ff4d6497a0cb6a177ad9f2_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:37,039 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/683d62f6b14347ca9894bf454abe1cb1, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:37,039 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/683d62f6b14347ca9894bf454abe1cb1 is 175, key is test_row_0/A:col10/1732340436957/Put/seqid=0 2024-11-23T05:40:37,082 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741959_1135 (size=39549) 2024-11-23T05:40:37,091 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=56, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/683d62f6b14347ca9894bf454abe1cb1 2024-11-23T05:40:37,113 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c167016c11a445fea1579ec06678084f is 50, key is test_row_0/B:col10/1732340436957/Put/seqid=0 2024-11-23T05:40:37,120 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741960_1136 (size=12001) 2024-11-23T05:40:37,121 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c167016c11a445fea1579ec06678084f 2024-11-23T05:40:37,134 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340497127, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,135 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,135 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340497128, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,144 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340497137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340497137, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,145 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340497138, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,149 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/026d3da8164d47d19376f48439423dcf is 50, key is test_row_0/C:col10/1732340436957/Put/seqid=0 2024-11-23T05:40:37,203 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741961_1137 (size=12001) 2024-11-23T05:40:37,347 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340497346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340497346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,348 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,359 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340497357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340497357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,360 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340497357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,361 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,362 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=42 2024-11-23T05:40:37,546 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:37,547 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees 2024-11-23T05:40:37,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T05:40:37,549 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:37,549 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=44, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:37,549 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=45, ppid=44, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:37,604 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=56 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/026d3da8164d47d19376f48439423dcf 2024-11-23T05:40:37,609 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/683d62f6b14347ca9894bf454abe1cb1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/683d62f6b14347ca9894bf454abe1cb1 2024-11-23T05:40:37,614 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/683d62f6b14347ca9894bf454abe1cb1, entries=200, sequenceid=56, filesize=38.6 K 2024-11-23T05:40:37,615 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c167016c11a445fea1579ec06678084f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c167016c11a445fea1579ec06678084f 2024-11-23T05:40:37,620 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c167016c11a445fea1579ec06678084f, entries=150, sequenceid=56, filesize=11.7 K 2024-11-23T05:40:37,622 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/026d3da8164d47d19376f48439423dcf as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/026d3da8164d47d19376f48439423dcf 2024-11-23T05:40:37,628 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/026d3da8164d47d19376f48439423dcf, entries=150, sequenceid=56, filesize=11.7 K 2024-11-23T05:40:37,630 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6fed8128b8c4dd3776e5696d683dbd53 in 672ms, sequenceid=56, compaction requested=true 2024-11-23T05:40:37,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:37,630 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:37,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:37,630 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:37,630 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:37,631 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:37,631 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101459 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:37,632 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:37,632 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,632 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/ef6a8452e33d469bb9d759463f60477f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a8ede2a63a19443db15e397b8e863b22, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/683d62f6b14347ca9894bf454abe1cb1] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=99.1 K 2024-11-23T05:40:37,632 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,632 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/ef6a8452e33d469bb9d759463f60477f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a8ede2a63a19443db15e397b8e863b22, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/683d62f6b14347ca9894bf454abe1cb1] 2024-11-23T05:40:37,633 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:37,633 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:37,633 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,633 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/72adc6dbbdeb42c8b9dafca84cd3c167, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0e1796d13d90473598561ca5ba1d8460, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c167016c11a445fea1579ec06678084f] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=35.2 K 2024-11-23T05:40:37,634 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 72adc6dbbdeb42c8b9dafca84cd3c167, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732340435415 2024-11-23T05:40:37,634 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ef6a8452e33d469bb9d759463f60477f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732340435415 2024-11-23T05:40:37,634 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0e1796d13d90473598561ca5ba1d8460, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732340435428 2024-11-23T05:40:37,634 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a8ede2a63a19443db15e397b8e863b22, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732340435428 2024-11-23T05:40:37,635 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting c167016c11a445fea1579ec06678084f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732340436619 2024-11-23T05:40:37,635 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 683d62f6b14347ca9894bf454abe1cb1, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732340436619 2024-11-23T05:40:37,649 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#119 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:37,649 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/44b3ae27e762478db34b38c88dc7c6cd is 50, key is test_row_0/B:col10/1732340436957/Put/seqid=0 2024-11-23T05:40:37,655 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:37,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T05:40:37,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:37,659 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:37,659 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T05:40:37,665 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112376efa7c12e754c84bb52c129867fa204_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:37,670 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112376efa7c12e754c84bb52c129867fa204_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:37,670 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112376efa7c12e754c84bb52c129867fa204_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:37,681 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741962_1138 (size=12104) 2024-11-23T05:40:37,686 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235ade59e7bcf240cd8b05a559a5c67881_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340437014/Put/seqid=0 2024-11-23T05:40:37,688 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,688 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340497679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340497679, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340497682, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,694 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340497693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,696 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,699 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,701 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741963_1139 (size=4469) 2024-11-23T05:40:37,701 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,702 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:37,702 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:37,703 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,704 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#120 average throughput is 0.51 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:37,704 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/e4787585633542078cde5625f0350e0b is 175, key is test_row_0/A:col10/1732340436957/Put/seqid=0 2024-11-23T05:40:37,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340497700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,705 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,709 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741964_1140 (size=14594) 2024-11-23T05:40:37,709 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:37,716 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411235ade59e7bcf240cd8b05a559a5c67881_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235ade59e7bcf240cd8b05a559a5c67881_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:37,717 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/366b7f5c99314e589c3ba35ebd2ae5c3, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:37,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/366b7f5c99314e589c3ba35ebd2ae5c3 is 175, key is test_row_0/A:col10/1732340437014/Put/seqid=0 2024-11-23T05:40:37,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741965_1141 (size=31058) 2024-11-23T05:40:37,743 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/e4787585633542078cde5625f0350e0b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/e4787585633542078cde5625f0350e0b 2024-11-23T05:40:37,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741966_1142 (size=39549) 2024-11-23T05:40:37,753 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into e4787585633542078cde5625f0350e0b(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:37,753 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:37,753 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=13, startTime=1732340437630; duration=0sec 2024-11-23T05:40:37,753 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:37,753 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:37,753 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:37,756 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:37,756 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:37,756 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,757 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/da01845e9cde4866bb3ef2373990191d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/514e815c87e54e66bfa3e7f30d53c008, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/026d3da8164d47d19376f48439423dcf] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=35.2 K 2024-11-23T05:40:37,757 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting da01845e9cde4866bb3ef2373990191d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=17, earliestPutTs=1732340435415 2024-11-23T05:40:37,757 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 514e815c87e54e66bfa3e7f30d53c008, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=41, earliestPutTs=1732340435428 2024-11-23T05:40:37,758 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 026d3da8164d47d19376f48439423dcf, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732340436619 2024-11-23T05:40:37,801 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#122 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:37,802 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/ffb5e51e3dbf4b8c9ee7664dca7800f0 is 50, key is test_row_0/C:col10/1732340436957/Put/seqid=0 2024-11-23T05:40:37,806 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340497806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,807 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,808 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340497806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,809 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340497806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,809 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,815 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340497806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,815 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,816 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,820 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:37,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340497816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:37,820 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:37,840 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741967_1143 (size=12104) 2024-11-23T05:40:37,852 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/ffb5e51e3dbf4b8c9ee7664dca7800f0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffb5e51e3dbf4b8c9ee7664dca7800f0 2024-11-23T05:40:37,858 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:37,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:37,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:37,858 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:37,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:37,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:37,863 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into ffb5e51e3dbf4b8c9ee7664dca7800f0(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:37,863 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:37,863 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=13, startTime=1732340437631; duration=0sec 2024-11-23T05:40:37,864 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:37,864 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:37,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T05:40:38,011 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:38,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:38,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,012 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,018 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340498016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,019 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,021 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340498017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,022 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,022 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340498017, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,023 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340498026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,033 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340498028, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,033 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,092 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/44b3ae27e762478db34b38c88dc7c6cd as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/44b3ae27e762478db34b38c88dc7c6cd 2024-11-23T05:40:38,106 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into 44b3ae27e762478db34b38c88dc7c6cd(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:38,106 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:38,106 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=13, startTime=1732340437630; duration=0sec 2024-11-23T05:40:38,106 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:38,106 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:38,150 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/366b7f5c99314e589c3ba35ebd2ae5c3 2024-11-23T05:40:38,164 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/9d4391a463294005b7e09e62b1899e0c is 50, key is test_row_0/B:col10/1732340437014/Put/seqid=0 2024-11-23T05:40:38,168 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:38,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,168 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:38,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,169 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,169 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T05:40:38,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741968_1144 (size=12001) 2024-11-23T05:40:38,185 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/9d4391a463294005b7e09e62b1899e0c 2024-11-23T05:40:38,201 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/1572f400aa824b729f763e5ac71cee1c is 50, key is test_row_0/C:col10/1732340437014/Put/seqid=0 2024-11-23T05:40:38,237 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741969_1145 (size=12001) 2024-11-23T05:40:38,322 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:38,324 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,326 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,330 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340498328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340498328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,331 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,331 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340498328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,332 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,333 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340498346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340498346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,481 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,482 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,482 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,635 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,636 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] handler.RSProcedureHandler(58): pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,636 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=45 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,637 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/1572f400aa824b729f763e5ac71cee1c 2024-11-23T05:40:38,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=45 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:38,645 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/366b7f5c99314e589c3ba35ebd2ae5c3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/366b7f5c99314e589c3ba35ebd2ae5c3 2024-11-23T05:40:38,653 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/366b7f5c99314e589c3ba35ebd2ae5c3, entries=200, sequenceid=78, filesize=38.6 K 2024-11-23T05:40:38,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/9d4391a463294005b7e09e62b1899e0c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9d4391a463294005b7e09e62b1899e0c 2024-11-23T05:40:38,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9d4391a463294005b7e09e62b1899e0c, entries=150, sequenceid=78, filesize=11.7 K 2024-11-23T05:40:38,670 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/1572f400aa824b729f763e5ac71cee1c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1572f400aa824b729f763e5ac71cee1c 2024-11-23T05:40:38,685 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1572f400aa824b729f763e5ac71cee1c, entries=150, sequenceid=78, filesize=11.7 K 2024-11-23T05:40:38,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T05:40:38,687 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6fed8128b8c4dd3776e5696d683dbd53 in 1028ms, sequenceid=78, compaction requested=false 2024-11-23T05:40:38,687 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:38,782 INFO [master/9abfc7c2569e:0.Chore.1 {}] migrate.RollingUpgradeChore(116): There is no table to migrate StoreFileTracker! 2024-11-23T05:40:38,782 INFO [master/9abfc7c2569e:0.Chore.1 {}] migrate.RollingUpgradeChore(85): All Rolling-Upgrade tasks are complete, shutdown RollingUpgradeChore! 2024-11-23T05:40:38,790 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=45 2024-11-23T05:40:38,790 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:38,790 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:40:38,805 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112380b6fa2ef0cb49f1b090f5216d59f10f_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340437679/Put/seqid=0 2024-11-23T05:40:38,819 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741970_1146 (size=12154) 2024-11-23T05:40:38,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:38,829 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112380b6fa2ef0cb49f1b090f5216d59f10f_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112380b6fa2ef0cb49f1b090f5216d59f10f_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:38,832 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96a73dfbf01e4cdda503ab66073197b5, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:38,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96a73dfbf01e4cdda503ab66073197b5 is 175, key is test_row_0/A:col10/1732340437679/Put/seqid=0 2024-11-23T05:40:38,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:38,840 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:38,842 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741971_1147 (size=30955) 2024-11-23T05:40:38,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,872 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340498865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340498866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,873 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340498870, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,878 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,878 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340498873, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340498871, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,879 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340498976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340498976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,982 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,983 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,987 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340498986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,988 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340498986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:38,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:38,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340498986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:38,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:38,990 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340499186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340499186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340499197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,199 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,199 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340499197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,200 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340499197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,200 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,243 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=95, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96a73dfbf01e4cdda503ab66073197b5 2024-11-23T05:40:39,254 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/84857c2ac9ac421f9d06847f96368bb2 is 50, key is test_row_0/B:col10/1732340437679/Put/seqid=0 2024-11-23T05:40:39,257 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741972_1148 (size=12001) 2024-11-23T05:40:39,259 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/84857c2ac9ac421f9d06847f96368bb2 2024-11-23T05:40:39,267 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/9e89293db84a4d4f99e5428bcdda9800 is 50, key is test_row_0/C:col10/1732340437679/Put/seqid=0 2024-11-23T05:40:39,270 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741973_1149 (size=12001) 2024-11-23T05:40:39,271 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=95 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/9e89293db84a4d4f99e5428bcdda9800 2024-11-23T05:40:39,276 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96a73dfbf01e4cdda503ab66073197b5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96a73dfbf01e4cdda503ab66073197b5 2024-11-23T05:40:39,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96a73dfbf01e4cdda503ab66073197b5, entries=150, sequenceid=95, filesize=30.2 K 2024-11-23T05:40:39,284 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/84857c2ac9ac421f9d06847f96368bb2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/84857c2ac9ac421f9d06847f96368bb2 2024-11-23T05:40:39,289 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/84857c2ac9ac421f9d06847f96368bb2, entries=150, sequenceid=95, filesize=11.7 K 2024-11-23T05:40:39,290 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/9e89293db84a4d4f99e5428bcdda9800 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9e89293db84a4d4f99e5428bcdda9800 2024-11-23T05:40:39,297 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9e89293db84a4d4f99e5428bcdda9800, entries=150, sequenceid=95, filesize=11.7 K 2024-11-23T05:40:39,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6fed8128b8c4dd3776e5696d683dbd53 in 508ms, sequenceid=95, compaction requested=true 2024-11-23T05:40:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:39,298 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=45}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=45 2024-11-23T05:40:39,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=45 2024-11-23T05:40:39,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=45, resume processing ppid=44 2024-11-23T05:40:39,304 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=45, ppid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7530 sec 2024-11-23T05:40:39,306 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=44, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=44, table=TestAcidGuarantees in 1.7570 sec 2024-11-23T05:40:39,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:39,500 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:39,500 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:40:39,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d597e632acab4f60b2cb239444c1fc00_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340439497/Put/seqid=0 2024-11-23T05:40:39,523 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340499516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,524 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340499517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,524 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340499517, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,524 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,525 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,525 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340499518, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,526 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340499519, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,526 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,531 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741974_1150 (size=14594) 2024-11-23T05:40:39,532 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:39,539 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d597e632acab4f60b2cb239444c1fc00_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d597e632acab4f60b2cb239444c1fc00_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:39,541 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5d95bd70e2be4663b53557730d1690fd, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:39,542 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5d95bd70e2be4663b53557730d1690fd is 175, key is test_row_0/A:col10/1732340439497/Put/seqid=0 2024-11-23T05:40:39,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741975_1151 (size=39549) 2024-11-23T05:40:39,549 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=118, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5d95bd70e2be4663b53557730d1690fd 2024-11-23T05:40:39,564 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/d4d961c923c94ceaace68a46c179380c is 50, key is test_row_0/B:col10/1732340439497/Put/seqid=0 2024-11-23T05:40:39,577 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741976_1152 (size=12001) 2024-11-23T05:40:39,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/d4d961c923c94ceaace68a46c179380c 2024-11-23T05:40:39,612 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/1ea175576e5d4111861bdc1529d51b79 is 50, key is test_row_0/C:col10/1732340439497/Put/seqid=0 2024-11-23T05:40:39,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741977_1153 (size=12001) 2024-11-23T05:40:39,625 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=118 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/1ea175576e5d4111861bdc1529d51b79 2024-11-23T05:40:39,631 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340499626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,631 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,634 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5d95bd70e2be4663b53557730d1690fd as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5d95bd70e2be4663b53557730d1690fd 2024-11-23T05:40:39,641 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340499636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,642 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340499637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,643 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,643 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340499637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,644 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340499638, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,644 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,645 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,646 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5d95bd70e2be4663b53557730d1690fd, entries=200, sequenceid=118, filesize=38.6 K 2024-11-23T05:40:39,649 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/d4d961c923c94ceaace68a46c179380c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/d4d961c923c94ceaace68a46c179380c 2024-11-23T05:40:39,654 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/d4d961c923c94ceaace68a46c179380c, entries=150, sequenceid=118, filesize=11.7 K 2024-11-23T05:40:39,655 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/1ea175576e5d4111861bdc1529d51b79 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1ea175576e5d4111861bdc1529d51b79 2024-11-23T05:40:39,661 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1ea175576e5d4111861bdc1529d51b79, entries=150, sequenceid=118, filesize=11.7 K 2024-11-23T05:40:39,662 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 6fed8128b8c4dd3776e5696d683dbd53 in 162ms, sequenceid=118, compaction requested=true 2024-11-23T05:40:39,662 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:39,662 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:39,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:39,665 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:39,665 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:39,666 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:39,666 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:39,666 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:39,666 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/e4787585633542078cde5625f0350e0b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/366b7f5c99314e589c3ba35ebd2ae5c3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96a73dfbf01e4cdda503ab66073197b5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5d95bd70e2be4663b53557730d1690fd] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=137.8 K 2024-11-23T05:40:39,666 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:39,667 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/e4787585633542078cde5625f0350e0b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/366b7f5c99314e589c3ba35ebd2ae5c3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96a73dfbf01e4cdda503ab66073197b5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5d95bd70e2be4663b53557730d1690fd] 2024-11-23T05:40:39,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:39,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:39,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:39,667 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:39,667 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:39,667 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:39,667 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:39,667 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/44b3ae27e762478db34b38c88dc7c6cd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9d4391a463294005b7e09e62b1899e0c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/84857c2ac9ac421f9d06847f96368bb2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/d4d961c923c94ceaace68a46c179380c] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=47.0 K 2024-11-23T05:40:39,668 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e4787585633542078cde5625f0350e0b, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732340436619 2024-11-23T05:40:39,668 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 44b3ae27e762478db34b38c88dc7c6cd, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732340436619 2024-11-23T05:40:39,668 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 366b7f5c99314e589c3ba35ebd2ae5c3, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340437004 2024-11-23T05:40:39,668 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96a73dfbf01e4cdda503ab66073197b5, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732340437679 2024-11-23T05:40:39,669 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d4391a463294005b7e09e62b1899e0c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340437014 2024-11-23T05:40:39,669 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5d95bd70e2be4663b53557730d1690fd, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732340438857 2024-11-23T05:40:39,669 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 84857c2ac9ac421f9d06847f96368bb2, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732340437679 2024-11-23T05:40:39,670 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d4d961c923c94ceaace68a46c179380c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732340438857 2024-11-23T05:40:39,687 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:39,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=44 2024-11-23T05:40:39,696 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:39,697 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#132 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:39,697 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:39,697 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/46e3f14bab3f447da33ddcbb70b873bc is 50, key is test_row_0/B:col10/1732340439497/Put/seqid=0 2024-11-23T05:40:39,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees 2024-11-23T05:40:39,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T05:40:39,700 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:39,701 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=46, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:39,701 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=47, ppid=46, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:39,707 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112328d5ae585d54435397c0977f737e0c62_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:39,718 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112328d5ae585d54435397c0977f737e0c62_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:39,719 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112328d5ae585d54435397c0977f737e0c62_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:39,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741978_1154 (size=12241) 2024-11-23T05:40:39,737 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/46e3f14bab3f447da33ddcbb70b873bc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/46e3f14bab3f447da33ddcbb70b873bc 2024-11-23T05:40:39,747 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into 46e3f14bab3f447da33ddcbb70b873bc(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:39,747 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:39,748 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=12, startTime=1732340439665; duration=0sec 2024-11-23T05:40:39,748 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:39,748 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:39,748 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:40:39,751 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:40:39,751 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:39,751 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:39,751 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffb5e51e3dbf4b8c9ee7664dca7800f0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1572f400aa824b729f763e5ac71cee1c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9e89293db84a4d4f99e5428bcdda9800, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1ea175576e5d4111861bdc1529d51b79] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=47.0 K 2024-11-23T05:40:39,752 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ffb5e51e3dbf4b8c9ee7664dca7800f0, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=56, earliestPutTs=1732340436619 2024-11-23T05:40:39,752 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1572f400aa824b729f763e5ac71cee1c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340437014 2024-11-23T05:40:39,753 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9e89293db84a4d4f99e5428bcdda9800, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=95, earliestPutTs=1732340437679 2024-11-23T05:40:39,754 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ea175576e5d4111861bdc1529d51b79, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732340438857 2024-11-23T05:40:39,763 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741979_1155 (size=4469) 2024-11-23T05:40:39,766 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#133 average throughput is 1.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:39,766 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/8ecb9147cf5d415c9fe7dffeb8518d96 is 50, key is test_row_0/C:col10/1732340439497/Put/seqid=0 2024-11-23T05:40:39,768 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#131 average throughput is 0.30 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:39,769 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/aa0056041813471090fd6f4014a6846e is 175, key is test_row_0/A:col10/1732340439497/Put/seqid=0 2024-11-23T05:40:39,791 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741981_1157 (size=31195) 2024-11-23T05:40:39,799 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/aa0056041813471090fd6f4014a6846e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/aa0056041813471090fd6f4014a6846e 2024-11-23T05:40:39,801 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741980_1156 (size=12241) 2024-11-23T05:40:39,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T05:40:39,811 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into aa0056041813471090fd6f4014a6846e(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:39,811 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:39,811 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=12, startTime=1732340439662; duration=0sec 2024-11-23T05:40:39,811 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:39,811 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:39,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:39,844 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:39,844 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T05:40:39,856 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:39,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:39,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:39,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:39,857 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:39,857 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:39,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:39,869 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c51d2c628eaa4499b086afaf07dabc20_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340439515/Put/seqid=0 2024-11-23T05:40:39,890 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340499882, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,891 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,891 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340499884, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,892 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,892 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340499887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,893 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340499887, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340499889, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,893 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,894 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,943 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741982_1158 (size=14744) 2024-11-23T05:40:39,997 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340499996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340499996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340499996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,998 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,998 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340499996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340499996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,009 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:40,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:40,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,010 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,010 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T05:40:40,163 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:40,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:40,163 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,163 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,164 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,164 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340500206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,208 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/8ecb9147cf5d415c9fe7dffeb8518d96 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/8ecb9147cf5d415c9fe7dffeb8518d96 2024-11-23T05:40:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,208 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340500206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340500207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,209 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340500207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,210 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340500208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,210 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,214 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into 8ecb9147cf5d415c9fe7dffeb8518d96(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:40,214 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:40,214 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=12, startTime=1732340439667; duration=0sec 2024-11-23T05:40:40,214 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:40,214 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:40,316 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:40,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:40,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,316 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,316 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T05:40:40,344 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:40,348 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c51d2c628eaa4499b086afaf07dabc20_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c51d2c628eaa4499b086afaf07dabc20_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:40,349 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/b4da424caff94cefadb7e4a7b240a97c, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:40,349 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/b4da424caff94cefadb7e4a7b240a97c is 175, key is test_row_0/A:col10/1732340439515/Put/seqid=0 2024-11-23T05:40:40,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741983_1159 (size=39699) 2024-11-23T05:40:40,469 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:40,469 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,469 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,470 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,516 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340500516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340500516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,517 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340500516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340500516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340500516, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:40,518 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:40,623 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:40,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,623 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,760 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=135, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/b4da424caff94cefadb7e4a7b240a97c 2024-11-23T05:40:40,768 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/7840faf403304c23b98a80e7ff8dfeaa is 50, key is test_row_0/B:col10/1732340439515/Put/seqid=0 2024-11-23T05:40:40,776 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741984_1160 (size=12101) 2024-11-23T05:40:40,776 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:40,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,776 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:40,777 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/7840faf403304c23b98a80e7ff8dfeaa 2024-11-23T05:40:40,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,777 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] handler.RSProcedureHandler(58): pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=47 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=47 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:40,785 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/b28f4e8e8ccd4d11879878704da11c39 is 50, key is test_row_0/C:col10/1732340439515/Put/seqid=0 2024-11-23T05:40:40,788 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741985_1161 (size=12101) 2024-11-23T05:40:40,789 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=135 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/b28f4e8e8ccd4d11879878704da11c39 2024-11-23T05:40:40,795 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/b4da424caff94cefadb7e4a7b240a97c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/b4da424caff94cefadb7e4a7b240a97c 2024-11-23T05:40:40,801 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/b4da424caff94cefadb7e4a7b240a97c, entries=200, sequenceid=135, filesize=38.8 K 2024-11-23T05:40:40,801 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/7840faf403304c23b98a80e7ff8dfeaa as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7840faf403304c23b98a80e7ff8dfeaa 2024-11-23T05:40:40,807 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7840faf403304c23b98a80e7ff8dfeaa, entries=150, sequenceid=135, filesize=11.8 K 2024-11-23T05:40:40,808 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/b28f4e8e8ccd4d11879878704da11c39 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b28f4e8e8ccd4d11879878704da11c39 2024-11-23T05:40:40,816 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b28f4e8e8ccd4d11879878704da11c39, entries=150, sequenceid=135, filesize=11.8 K 2024-11-23T05:40:40,817 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 6fed8128b8c4dd3776e5696d683dbd53 in 973ms, sequenceid=135, compaction requested=false 2024-11-23T05:40:40,817 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:40,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T05:40:40,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=47 2024-11-23T05:40:40,930 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:40,930 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T05:40:40,940 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b433cfae874c4fa2bbba409fc1c6833d_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340439884/Put/seqid=0 2024-11-23T05:40:40,950 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741986_1162 (size=12304) 2024-11-23T05:40:41,028 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:41,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340501037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340501039, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340501040, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,045 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,045 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340501042, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340501044, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,046 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340501155, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340501156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340501156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,159 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,159 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340501156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,160 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,160 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340501156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,160 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,350 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:41,354 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b433cfae874c4fa2bbba409fc1c6833d_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b433cfae874c4fa2bbba409fc1c6833d_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:41,355 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96be78beb8c2479b88008097ae0e9b54, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:41,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96be78beb8c2479b88008097ae0e9b54 is 175, key is test_row_0/A:col10/1732340439884/Put/seqid=0 2024-11-23T05:40:41,359 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741987_1163 (size=31105) 2024-11-23T05:40:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340501366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,367 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340501366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340501366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,368 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340501367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340501367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,370 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340501675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,678 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,678 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340501676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340501676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340501676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:41,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340501676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,679 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,680 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:41,760 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=40.3 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96be78beb8c2479b88008097ae0e9b54 2024-11-23T05:40:41,768 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/7151e5f859f64ec79ff8ccb8a8a4677a is 50, key is test_row_0/B:col10/1732340439884/Put/seqid=0 2024-11-23T05:40:41,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741988_1164 (size=12151) 2024-11-23T05:40:41,772 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/7151e5f859f64ec79ff8ccb8a8a4677a 2024-11-23T05:40:41,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/e0b579c7c46240cdbb6856b5ac4e937a is 50, key is test_row_0/C:col10/1732340439884/Put/seqid=0 2024-11-23T05:40:41,786 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741989_1165 (size=12151) 2024-11-23T05:40:41,790 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/e0b579c7c46240cdbb6856b5ac4e937a 2024-11-23T05:40:41,800 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/96be78beb8c2479b88008097ae0e9b54 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96be78beb8c2479b88008097ae0e9b54 2024-11-23T05:40:41,806 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96be78beb8c2479b88008097ae0e9b54, entries=150, sequenceid=157, filesize=30.4 K 2024-11-23T05:40:41,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/7151e5f859f64ec79ff8ccb8a8a4677a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7151e5f859f64ec79ff8ccb8a8a4677a 2024-11-23T05:40:41,812 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7151e5f859f64ec79ff8ccb8a8a4677a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-23T05:40:41,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/e0b579c7c46240cdbb6856b5ac4e937a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/e0b579c7c46240cdbb6856b5ac4e937a 2024-11-23T05:40:41,821 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/e0b579c7c46240cdbb6856b5ac4e937a, entries=150, sequenceid=157, filesize=11.9 K 2024-11-23T05:40:41,822 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 6fed8128b8c4dd3776e5696d683dbd53 in 892ms, sequenceid=157, compaction requested=true 2024-11-23T05:40:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:41,822 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=47}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=47 2024-11-23T05:40:41,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=47 2024-11-23T05:40:41,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=47, resume processing ppid=46 2024-11-23T05:40:41,826 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=47, ppid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1230 sec 2024-11-23T05:40:41,827 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=46, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=46, table=TestAcidGuarantees in 2.1290 sec 2024-11-23T05:40:41,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=46 2024-11-23T05:40:41,846 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:41,846 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:41,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees 2024-11-23T05:40:41,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T05:40:41,848 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:41,849 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=48, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:41,849 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=49, ppid=48, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:41,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T05:40:42,003 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=49 2024-11-23T05:40:42,003 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:42,003 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T05:40:42,011 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112323f73eb39f2e431eae6591c1f5ef3c1e_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340441043/Put/seqid=0 2024-11-23T05:40:42,014 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741990_1166 (size=12304) 2024-11-23T05:40:42,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T05:40:42,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:42,187 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:42,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340502232, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,238 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340502237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,242 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340502237, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340502238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,243 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,243 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340502238, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,244 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,347 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340502346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,347 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340502346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,349 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340502347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,349 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340502347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,350 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,353 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340502347, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,354 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,415 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:42,419 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112323f73eb39f2e431eae6591c1f5ef3c1e_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112323f73eb39f2e431eae6591c1f5ef3c1e_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:42,420 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a3c4a003818d414a9799a039a94e3bc0, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:42,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a3c4a003818d414a9799a039a94e3bc0 is 175, key is test_row_0/A:col10/1732340441043/Put/seqid=0 2024-11-23T05:40:42,425 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741991_1167 (size=31105) 2024-11-23T05:40:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T05:40:42,561 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340502555, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340502556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,562 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340502556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,562 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,562 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340502556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,563 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340502556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,563 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,826 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=172, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a3c4a003818d414a9799a039a94e3bc0 2024-11-23T05:40:42,833 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/8a5c3c81cd1e4b01bbf939a3bc1e4155 is 50, key is test_row_0/B:col10/1732340441043/Put/seqid=0 2024-11-23T05:40:42,837 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741992_1168 (size=12151) 2024-11-23T05:40:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340502866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,869 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,869 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340502866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340502866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340502866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,870 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340502866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:42,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:42,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T05:40:43,238 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/8a5c3c81cd1e4b01bbf939a3bc1e4155 2024-11-23T05:40:43,246 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/fa7a08a0a7d84c1d846a2ed0714b875f is 50, key is test_row_0/C:col10/1732340441043/Put/seqid=0 2024-11-23T05:40:43,252 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741993_1169 (size=12151) 2024-11-23T05:40:43,252 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/fa7a08a0a7d84c1d846a2ed0714b875f 2024-11-23T05:40:43,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/a3c4a003818d414a9799a039a94e3bc0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a3c4a003818d414a9799a039a94e3bc0 2024-11-23T05:40:43,265 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a3c4a003818d414a9799a039a94e3bc0, entries=150, sequenceid=172, filesize=30.4 K 2024-11-23T05:40:43,266 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/8a5c3c81cd1e4b01bbf939a3bc1e4155 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/8a5c3c81cd1e4b01bbf939a3bc1e4155 2024-11-23T05:40:43,271 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/8a5c3c81cd1e4b01bbf939a3bc1e4155, entries=150, sequenceid=172, filesize=11.9 K 2024-11-23T05:40:43,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/fa7a08a0a7d84c1d846a2ed0714b875f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa7a08a0a7d84c1d846a2ed0714b875f 2024-11-23T05:40:43,280 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa7a08a0a7d84c1d846a2ed0714b875f, entries=150, sequenceid=172, filesize=11.9 K 2024-11-23T05:40:43,281 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=127.47 KB/130530 for 6fed8128b8c4dd3776e5696d683dbd53 in 1278ms, sequenceid=172, compaction requested=true 2024-11-23T05:40:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:43,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=49}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=49 2024-11-23T05:40:43,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=49 2024-11-23T05:40:43,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=49, resume processing ppid=48 2024-11-23T05:40:43,285 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=49, ppid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4340 sec 2024-11-23T05:40:43,290 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=48, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=48, table=TestAcidGuarantees in 1.4400 sec 2024-11-23T05:40:43,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:43,378 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:43,378 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:40:43,386 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237c92612dd7014e27a85ed6e1b5669816_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340443377/Put/seqid=0 2024-11-23T05:40:43,398 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741994_1170 (size=14794) 2024-11-23T05:40:43,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340503391, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,423 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340503392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,423 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340503422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,429 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340503422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,430 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340503422, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,430 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,529 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340503526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340503526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,529 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,540 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340503535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340503535, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,541 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340503536, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,541 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,542 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,738 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340503736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340503736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,738 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340503746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,751 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340503746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,752 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340503746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:43,752 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:43,799 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:43,802 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237c92612dd7014e27a85ed6e1b5669816_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237c92612dd7014e27a85ed6e1b5669816_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:43,803 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/62177c0df8e1496ab7148f2d6ac42e8b, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:43,804 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/62177c0df8e1496ab7148f2d6ac42e8b is 175, key is test_row_0/A:col10/1732340443377/Put/seqid=0 2024-11-23T05:40:43,814 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741995_1171 (size=39749) 2024-11-23T05:40:43,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=48 2024-11-23T05:40:43,996 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:43,997 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees 2024-11-23T05:40:43,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T05:40:43,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:43,999 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=50, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:43,999 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=51, ppid=50, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340504045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,046 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340504045, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,047 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,059 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340504056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340504056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,059 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,059 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,060 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340504056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,060 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T05:40:44,151 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T05:40:44,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,151 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:44,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,152 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,152 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,208 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=195, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/62177c0df8e1496ab7148f2d6ac42e8b 2024-11-23T05:40:44,218 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/5517a513958047e388882c61b1affcfb is 50, key is test_row_0/B:col10/1732340443377/Put/seqid=0 2024-11-23T05:40:44,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741996_1172 (size=12151) 2024-11-23T05:40:44,303 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T05:40:44,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:44,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,304 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,304 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T05:40:44,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T05:40:44,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:44,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,558 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340504556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340504556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,559 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,560 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,571 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340504566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,572 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,572 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340504566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340504566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:44,573 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:44,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T05:40:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,609 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:44,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,622 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/5517a513958047e388882c61b1affcfb 2024-11-23T05:40:44,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T05:40:44,630 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/0a62db71fcb64c9eb6b00802ab655bee is 50, key is test_row_0/C:col10/1732340443377/Put/seqid=0 2024-11-23T05:40:44,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741997_1173 (size=12151) 2024-11-23T05:40:44,763 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T05:40:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,763 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,763 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,916 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T05:40:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:44,916 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:44,916 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] handler.RSProcedureHandler(58): pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=51 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:44,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=51 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:45,036 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/0a62db71fcb64c9eb6b00802ab655bee 2024-11-23T05:40:45,044 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/62177c0df8e1496ab7148f2d6ac42e8b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/62177c0df8e1496ab7148f2d6ac42e8b 2024-11-23T05:40:45,050 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/62177c0df8e1496ab7148f2d6ac42e8b, entries=200, sequenceid=195, filesize=38.8 K 2024-11-23T05:40:45,053 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/5517a513958047e388882c61b1affcfb as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/5517a513958047e388882c61b1affcfb 2024-11-23T05:40:45,059 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/5517a513958047e388882c61b1affcfb, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T05:40:45,062 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/0a62db71fcb64c9eb6b00802ab655bee as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/0a62db71fcb64c9eb6b00802ab655bee 2024-11-23T05:40:45,069 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/0a62db71fcb64c9eb6b00802ab655bee, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T05:40:45,070 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 6fed8128b8c4dd3776e5696d683dbd53 in 1692ms, sequenceid=195, compaction requested=true 2024-11-23T05:40:45,070 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:45,071 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T05:40:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:45,071 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T05:40:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:45,071 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:45,072 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:45,073 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60795 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T05:40:45,073 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:45,073 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:45,073 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/46e3f14bab3f447da33ddcbb70b873bc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7840faf403304c23b98a80e7ff8dfeaa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7151e5f859f64ec79ff8ccb8a8a4677a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/8a5c3c81cd1e4b01bbf939a3bc1e4155, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/5517a513958047e388882c61b1affcfb] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=59.4 K 2024-11-23T05:40:45,073 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 172853 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T05:40:45,073 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:45,073 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:45,074 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/aa0056041813471090fd6f4014a6846e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/b4da424caff94cefadb7e4a7b240a97c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96be78beb8c2479b88008097ae0e9b54, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a3c4a003818d414a9799a039a94e3bc0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/62177c0df8e1496ab7148f2d6ac42e8b] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=168.8 K 2024-11-23T05:40:45,074 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=11 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:45,074 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/aa0056041813471090fd6f4014a6846e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/b4da424caff94cefadb7e4a7b240a97c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96be78beb8c2479b88008097ae0e9b54, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a3c4a003818d414a9799a039a94e3bc0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/62177c0df8e1496ab7148f2d6ac42e8b] 2024-11-23T05:40:45,074 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 46e3f14bab3f447da33ddcbb70b873bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732340438857 2024-11-23T05:40:45,074 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting aa0056041813471090fd6f4014a6846e, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732340438857 2024-11-23T05:40:45,074 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7840faf403304c23b98a80e7ff8dfeaa, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732340439515 2024-11-23T05:40:45,075 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b4da424caff94cefadb7e4a7b240a97c, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732340439515 2024-11-23T05:40:45,075 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7151e5f859f64ec79ff8ccb8a8a4677a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340439883 2024-11-23T05:40:45,075 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 96be78beb8c2479b88008097ae0e9b54, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340439883 2024-11-23T05:40:45,076 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8a5c3c81cd1e4b01bbf939a3bc1e4155, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732340441037 2024-11-23T05:40:45,076 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a3c4a003818d414a9799a039a94e3bc0, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732340441037 2024-11-23T05:40:45,076 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5517a513958047e388882c61b1affcfb, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340442236 2024-11-23T05:40:45,076 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 62177c0df8e1496ab7148f2d6ac42e8b, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340442235 2024-11-23T05:40:45,090 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:45,094 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#147 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:45,095 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/9b660f44a2fa4ccfbfb67b3a2f6f4092 is 50, key is test_row_0/B:col10/1732340443377/Put/seqid=0 2024-11-23T05:40:45,095 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411238c609f51f016469d8cfe115beadadee1_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:45,098 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411238c609f51f016469d8cfe115beadadee1_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:45,099 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411238c609f51f016469d8cfe115beadadee1_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:45,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=51 2024-11-23T05:40:45,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:45,109 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:40:45,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112300156361806342ed8a5bae744d20eb74_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340443389/Put/seqid=0 2024-11-23T05:40:45,123 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741999_1175 (size=4469) 2024-11-23T05:40:45,124 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#146 average throughput is 0.72 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:45,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741998_1174 (size=12561) 2024-11-23T05:40:45,126 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/c4b9e01777b34b0c8d7aa8d5aad363c3 is 175, key is test_row_0/A:col10/1732340443377/Put/seqid=0 2024-11-23T05:40:45,127 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742000_1176 (size=12304) 2024-11-23T05:40:45,132 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742001_1177 (size=31515) 2024-11-23T05:40:45,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T05:40:45,142 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/c4b9e01777b34b0c8d7aa8d5aad363c3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c4b9e01777b34b0c8d7aa8d5aad363c3 2024-11-23T05:40:45,148 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into c4b9e01777b34b0c8d7aa8d5aad363c3(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:45,148 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:45,148 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=11, startTime=1732340445071; duration=0sec 2024-11-23T05:40:45,149 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:45,149 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:45,149 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 5 store files, 0 compacting, 5 eligible, 16 blocking 2024-11-23T05:40:45,150 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 5 files of size 60795 starting at candidate #0 after considering 6 permutations with 6 in ratio 2024-11-23T05:40:45,151 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:45,151 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:45,151 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/8ecb9147cf5d415c9fe7dffeb8518d96, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b28f4e8e8ccd4d11879878704da11c39, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/e0b579c7c46240cdbb6856b5ac4e937a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa7a08a0a7d84c1d846a2ed0714b875f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/0a62db71fcb64c9eb6b00802ab655bee] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=59.4 K 2024-11-23T05:40:45,151 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8ecb9147cf5d415c9fe7dffeb8518d96, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=118, earliestPutTs=1732340438857 2024-11-23T05:40:45,152 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b28f4e8e8ccd4d11879878704da11c39, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=135, earliestPutTs=1732340439515 2024-11-23T05:40:45,152 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0b579c7c46240cdbb6856b5ac4e937a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340439883 2024-11-23T05:40:45,152 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting fa7a08a0a7d84c1d846a2ed0714b875f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732340441037 2024-11-23T05:40:45,153 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0a62db71fcb64c9eb6b00802ab655bee, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340442236 2024-11-23T05:40:45,165 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#149 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:45,166 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/fa81dc225e3845d1bea4e505812bfdb4 is 50, key is test_row_0/C:col10/1732340443377/Put/seqid=0 2024-11-23T05:40:45,170 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742002_1178 (size=12561) 2024-11-23T05:40:45,185 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/fa81dc225e3845d1bea4e505812bfdb4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa81dc225e3845d1bea4e505812bfdb4 2024-11-23T05:40:45,193 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into fa81dc225e3845d1bea4e505812bfdb4(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:45,193 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:45,195 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=11, startTime=1732340445071; duration=0sec 2024-11-23T05:40:45,195 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:45,195 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:45,529 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:45,536 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/9b660f44a2fa4ccfbfb67b3a2f6f4092 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9b660f44a2fa4ccfbfb67b3a2f6f4092 2024-11-23T05:40:45,538 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112300156361806342ed8a5bae744d20eb74_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112300156361806342ed8a5bae744d20eb74_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:45,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/fdaa816e3a9945d8ad405329f20cd811, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:45,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/fdaa816e3a9945d8ad405329f20cd811 is 175, key is test_row_0/A:col10/1732340443389/Put/seqid=0 2024-11-23T05:40:45,549 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 5 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into 9b660f44a2fa4ccfbfb67b3a2f6f4092(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:45,549 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:45,549 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=11, startTime=1732340445071; duration=0sec 2024-11-23T05:40:45,549 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:45,549 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:45,569 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742003_1179 (size=31105) 2024-11-23T05:40:45,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:45,570 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:45,571 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=209, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/fdaa816e3a9945d8ad405329f20cd811 2024-11-23T05:40:45,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/ebd9be39689e47bdaa289e26dfc180b7 is 50, key is test_row_0/B:col10/1732340443389/Put/seqid=0 2024-11-23T05:40:45,606 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340505601, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,606 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340505603, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,609 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340505604, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340505606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,610 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340505606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,610 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,611 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742004_1180 (size=12151) 2024-11-23T05:40:45,613 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/ebd9be39689e47bdaa289e26dfc180b7 2024-11-23T05:40:45,623 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/d0c76d4ec1db48fab6197920a833b29f is 50, key is test_row_0/C:col10/1732340443389/Put/seqid=0 2024-11-23T05:40:45,644 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742005_1181 (size=12151) 2024-11-23T05:40:45,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340505716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340505717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340505716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,720 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,720 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340505717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340505717, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,721 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340505926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340505926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340505926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340505928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340505928, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,931 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:45,932 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,045 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=209 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/d0c76d4ec1db48fab6197920a833b29f 2024-11-23T05:40:46,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/fdaa816e3a9945d8ad405329f20cd811 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/fdaa816e3a9945d8ad405329f20cd811 2024-11-23T05:40:46,061 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/fdaa816e3a9945d8ad405329f20cd811, entries=150, sequenceid=209, filesize=30.4 K 2024-11-23T05:40:46,064 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/ebd9be39689e47bdaa289e26dfc180b7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/ebd9be39689e47bdaa289e26dfc180b7 2024-11-23T05:40:46,075 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/ebd9be39689e47bdaa289e26dfc180b7, entries=150, sequenceid=209, filesize=11.9 K 2024-11-23T05:40:46,076 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/d0c76d4ec1db48fab6197920a833b29f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/d0c76d4ec1db48fab6197920a833b29f 2024-11-23T05:40:46,082 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/d0c76d4ec1db48fab6197920a833b29f, entries=150, sequenceid=209, filesize=11.9 K 2024-11-23T05:40:46,083 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6fed8128b8c4dd3776e5696d683dbd53 in 974ms, sequenceid=209, compaction requested=false 2024-11-23T05:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,083 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=51}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=51 2024-11-23T05:40:46,084 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=51 2024-11-23T05:40:46,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=51, resume processing ppid=50 2024-11-23T05:40:46,087 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=51, ppid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0860 sec 2024-11-23T05:40:46,089 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=50, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=50, table=TestAcidGuarantees in 2.0900 sec 2024-11-23T05:40:46,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=50 2024-11-23T05:40:46,146 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:46,146 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:46,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees 2024-11-23T05:40:46,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T05:40:46,148 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:46,149 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=52, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:46,149 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=53, ppid=52, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:46,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:46,242 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:46,242 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:40:46,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T05:40:46,281 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232fa4cd82cd374e6aac8b8863ec7a18ec_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340445595/Put/seqid=0 2024-11-23T05:40:46,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340506278, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340506287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,294 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340506287, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,294 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,295 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340506289, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,295 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340506293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,297 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:46,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,303 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,303 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,332 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742006_1182 (size=14794) 2024-11-23T05:40:46,400 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340506396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,401 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340506396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,401 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,402 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,406 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340506406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,407 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,410 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340506406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340506406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,413 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,456 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:46,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,456 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,457 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T05:40:46,607 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340506605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,608 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340506606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,608 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,609 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:46,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:46,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,610 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,616 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,616 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340506616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340506616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,617 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,628 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340506627, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,629 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,733 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:46,738 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232fa4cd82cd374e6aac8b8863ec7a18ec_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232fa4cd82cd374e6aac8b8863ec7a18ec_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:46,739 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75d9f8d1f4af4e63997fb4c786326d81, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:46,739 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75d9f8d1f4af4e63997fb4c786326d81 is 175, key is test_row_0/A:col10/1732340445595/Put/seqid=0 2024-11-23T05:40:46,750 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742007_1183 (size=39749) 2024-11-23T05:40:46,752 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=235, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75d9f8d1f4af4e63997fb4c786326d81 2024-11-23T05:40:46,764 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/316aaf3d2bc3446bb7d743d6fe711d2c is 50, key is test_row_0/B:col10/1732340445595/Put/seqid=0 2024-11-23T05:40:46,765 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:46,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:46,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T05:40:46,811 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742008_1184 (size=12151) 2024-11-23T05:40:46,812 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/316aaf3d2bc3446bb7d743d6fe711d2c 2024-11-23T05:40:46,826 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/9cadfb8e0c534d4db8a51936fc39067f is 50, key is test_row_0/C:col10/1732340445595/Put/seqid=0 2024-11-23T05:40:46,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742009_1185 (size=12151) 2024-11-23T05:40:46,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/9cadfb8e0c534d4db8a51936fc39067f 2024-11-23T05:40:46,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75d9f8d1f4af4e63997fb4c786326d81 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75d9f8d1f4af4e63997fb4c786326d81 2024-11-23T05:40:46,899 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75d9f8d1f4af4e63997fb4c786326d81, entries=200, sequenceid=235, filesize=38.8 K 2024-11-23T05:40:46,906 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/316aaf3d2bc3446bb7d743d6fe711d2c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/316aaf3d2bc3446bb7d743d6fe711d2c 2024-11-23T05:40:46,920 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:46,920 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,920 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,921 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:46,922 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/316aaf3d2bc3446bb7d743d6fe711d2c, entries=150, sequenceid=235, filesize=11.9 K 2024-11-23T05:40:46,922 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340506916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,923 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:46,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340506916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:46,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/9cadfb8e0c534d4db8a51936fc39067f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9cadfb8e0c534d4db8a51936fc39067f 2024-11-23T05:40:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:46,924 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:46,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9cadfb8e0c534d4db8a51936fc39067f, entries=150, sequenceid=235, filesize=11.9 K 2024-11-23T05:40:46,932 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 6fed8128b8c4dd3776e5696d683dbd53 in 690ms, sequenceid=235, compaction requested=true 2024-11-23T05:40:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:46,932 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:46,932 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:46,932 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:46,933 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 102369 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:46,933 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:46,934 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:46,934 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:46,934 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,934 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,934 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c4b9e01777b34b0c8d7aa8d5aad363c3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/fdaa816e3a9945d8ad405329f20cd811, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75d9f8d1f4af4e63997fb4c786326d81] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=100.0 K 2024-11-23T05:40:46,934 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9b660f44a2fa4ccfbfb67b3a2f6f4092, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/ebd9be39689e47bdaa289e26dfc180b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/316aaf3d2bc3446bb7d743d6fe711d2c] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.0 K 2024-11-23T05:40:46,934 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:46,934 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c4b9e01777b34b0c8d7aa8d5aad363c3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/fdaa816e3a9945d8ad405329f20cd811, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75d9f8d1f4af4e63997fb4c786326d81] 2024-11-23T05:40:46,935 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9b660f44a2fa4ccfbfb67b3a2f6f4092, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340442236 2024-11-23T05:40:46,935 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c4b9e01777b34b0c8d7aa8d5aad363c3, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340442236 2024-11-23T05:40:46,935 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting fdaa816e3a9945d8ad405329f20cd811, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732340443389 2024-11-23T05:40:46,935 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ebd9be39689e47bdaa289e26dfc180b7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732340443389 2024-11-23T05:40:46,936 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 316aaf3d2bc3446bb7d743d6fe711d2c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340445595 2024-11-23T05:40:46,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75d9f8d1f4af4e63997fb4c786326d81, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340445595 2024-11-23T05:40:46,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:46,948 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:46,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T05:40:46,954 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#155 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:46,954 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/e581eca25bfc4913bd6300c236ad078d is 50, key is test_row_0/B:col10/1732340445595/Put/seqid=0 2024-11-23T05:40:46,967 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:46,984 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123844ecc29fdfc4d368cae8e7090a8feb9_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:46,986 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123844ecc29fdfc4d368cae8e7090a8feb9_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:46,986 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123844ecc29fdfc4d368cae8e7090a8feb9_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:46,989 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ef550911c11a4d62bfbf4c9d2f536386_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340446946/Put/seqid=0 2024-11-23T05:40:46,996 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742010_1186 (size=12663) 2024-11-23T05:40:47,004 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742011_1187 (size=4469) 2024-11-23T05:40:47,014 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340507006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,015 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,020 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340507015, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,021 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340507013, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,022 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,022 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742012_1188 (size=14794) 2024-11-23T05:40:47,074 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:47,074 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,074 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,075 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,129 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340507126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,129 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,130 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340507126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,131 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,131 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340507126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,228 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,229 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,229 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T05:40:47,337 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340507336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,338 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340507336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,338 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,339 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340507346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,353 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,384 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:47,384 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:47,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,385 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,402 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/e581eca25bfc4913bd6300c236ad078d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e581eca25bfc4913bd6300c236ad078d 2024-11-23T05:40:47,411 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#156 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:47,412 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/05cf0049e43c440ebe431545ed689633 is 175, key is test_row_0/A:col10/1732340445595/Put/seqid=0 2024-11-23T05:40:47,420 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742013_1189 (size=31617) 2024-11-23T05:40:47,420 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into e581eca25bfc4913bd6300c236ad078d(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:47,420 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:47,420 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=13, startTime=1732340446932; duration=0sec 2024-11-23T05:40:47,420 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:47,420 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:47,420 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:47,423 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:47,423 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36863 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:47,423 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:47,423 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,424 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa81dc225e3845d1bea4e505812bfdb4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/d0c76d4ec1db48fab6197920a833b29f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9cadfb8e0c534d4db8a51936fc39067f] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.0 K 2024-11-23T05:40:47,424 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting fa81dc225e3845d1bea4e505812bfdb4, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340442236 2024-11-23T05:40:47,425 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d0c76d4ec1db48fab6197920a833b29f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=209, earliestPutTs=1732340443389 2024-11-23T05:40:47,425 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9cadfb8e0c534d4db8a51936fc39067f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340445595 2024-11-23T05:40:47,427 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/05cf0049e43c440ebe431545ed689633 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/05cf0049e43c440ebe431545ed689633 2024-11-23T05:40:47,427 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ef550911c11a4d62bfbf4c9d2f536386_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ef550911c11a4d62bfbf4c9d2f536386_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:47,428 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75a0d4a2b2be47e49ecee8de4f5bcdea, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:47,428 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75a0d4a2b2be47e49ecee8de4f5bcdea is 175, key is test_row_0/A:col10/1732340446946/Put/seqid=0 2024-11-23T05:40:47,432 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into 05cf0049e43c440ebe431545ed689633(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:47,432 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:47,432 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=13, startTime=1732340446932; duration=0sec 2024-11-23T05:40:47,432 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:47,432 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:47,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340507431, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,440 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340507436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,441 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742014_1190 (size=39749) 2024-11-23T05:40:47,442 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#158 average throughput is 2.18 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:47,442 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/3184060ed6a34116a55f7570bb642ea9 is 50, key is test_row_0/C:col10/1732340445595/Put/seqid=0 2024-11-23T05:40:47,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742015_1191 (size=12663) 2024-11-23T05:40:47,537 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:47,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:47,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,538 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340507646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,647 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,647 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340507646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,648 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340507656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,657 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:47,658 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:47,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:47,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,692 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,843 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=251, memsize=29.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75a0d4a2b2be47e49ecee8de4f5bcdea 2024-11-23T05:40:47,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:47,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:47,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,846 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:47,863 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/3184060ed6a34116a55f7570bb642ea9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3184060ed6a34116a55f7570bb642ea9 2024-11-23T05:40:47,869 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into 3184060ed6a34116a55f7570bb642ea9(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:47,869 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:47,869 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=13, startTime=1732340446932; duration=0sec 2024-11-23T05:40:47,869 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:47,869 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:47,876 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/395156a5568047e2bae6d10cd47ede6f is 50, key is test_row_0/B:col10/1732340446946/Put/seqid=0 2024-11-23T05:40:47,909 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742016_1192 (size=12151) 2024-11-23T05:40:47,913 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/395156a5568047e2bae6d10cd47ede6f 2024-11-23T05:40:47,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/b3da6d269e234959821ab56114edf2e0 is 50, key is test_row_0/C:col10/1732340446946/Put/seqid=0 2024-11-23T05:40:47,936 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742017_1193 (size=12151) 2024-11-23T05:40:47,999 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:47,999 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:47,999 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,000 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,152 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,153 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,153 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,154 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340508156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,157 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340508156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,158 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340508166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,168 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,169 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T05:40:48,307 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,307 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,307 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,337 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/b3da6d269e234959821ab56114edf2e0 2024-11-23T05:40:48,342 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/75a0d4a2b2be47e49ecee8de4f5bcdea as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75a0d4a2b2be47e49ecee8de4f5bcdea 2024-11-23T05:40:48,346 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75a0d4a2b2be47e49ecee8de4f5bcdea, entries=200, sequenceid=251, filesize=38.8 K 2024-11-23T05:40:48,347 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/395156a5568047e2bae6d10cd47ede6f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/395156a5568047e2bae6d10cd47ede6f 2024-11-23T05:40:48,351 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/395156a5568047e2bae6d10cd47ede6f, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T05:40:48,352 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/b3da6d269e234959821ab56114edf2e0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b3da6d269e234959821ab56114edf2e0 2024-11-23T05:40:48,356 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b3da6d269e234959821ab56114edf2e0, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T05:40:48,357 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 6fed8128b8c4dd3776e5696d683dbd53 in 1409ms, sequenceid=251, compaction requested=false 2024-11-23T05:40:48,357 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:48,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:48,458 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:48,458 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T05:40:48,460 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,460 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,460 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,467 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123950e712c41fd4bb99f2c0c57981f9ed8_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340448458/Put/seqid=0 2024-11-23T05:40:48,473 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742018_1194 (size=14994) 2024-11-23T05:40:48,473 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:48,478 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123950e712c41fd4bb99f2c0c57981f9ed8_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123950e712c41fd4bb99f2c0c57981f9ed8_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:48,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5a64a6f1b0284c59a796117eeec61f0a, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:48,479 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5a64a6f1b0284c59a796117eeec61f0a is 175, key is test_row_0/A:col10/1732340448458/Put/seqid=0 2024-11-23T05:40:48,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340508484, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,485 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340508485, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,485 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,486 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742019_1195 (size=39949) 2024-11-23T05:40:48,492 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5a64a6f1b0284c59a796117eeec61f0a 2024-11-23T05:40:48,502 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8 is 50, key is test_row_0/B:col10/1732340448458/Put/seqid=0 2024-11-23T05:40:48,515 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742020_1196 (size=12301) 2024-11-23T05:40:48,521 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8 2024-11-23T05:40:48,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/ffba094c8f6348fc8747c3c19d157249 is 50, key is test_row_0/C:col10/1732340448458/Put/seqid=0 2024-11-23T05:40:48,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742021_1197 (size=12301) 2024-11-23T05:40:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,596 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340508596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340508596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,613 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,613 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,766 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:48,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:48,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,766 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,766 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,807 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340508806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340508805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,808 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:48,919 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,919 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] handler.RSProcedureHandler(58): pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,919 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=53 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=53 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:48,937 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/ffba094c8f6348fc8747c3c19d157249 2024-11-23T05:40:48,942 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/5a64a6f1b0284c59a796117eeec61f0a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5a64a6f1b0284c59a796117eeec61f0a 2024-11-23T05:40:48,947 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5a64a6f1b0284c59a796117eeec61f0a, entries=200, sequenceid=276, filesize=39.0 K 2024-11-23T05:40:48,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8 2024-11-23T05:40:48,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8, entries=150, sequenceid=276, filesize=12.0 K 2024-11-23T05:40:48,955 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/ffba094c8f6348fc8747c3c19d157249 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffba094c8f6348fc8747c3c19d157249 2024-11-23T05:40:48,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffba094c8f6348fc8747c3c19d157249, entries=150, sequenceid=276, filesize=12.0 K 2024-11-23T05:40:48,960 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for 6fed8128b8c4dd3776e5696d683dbd53 in 502ms, sequenceid=276, compaction requested=true 2024-11-23T05:40:48,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:48,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:48,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:48,960 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:48,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:48,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:48,960 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:48,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:48,960 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:48,962 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 111315 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:48,962 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:48,962 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:48,962 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:48,962 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,962 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,962 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/05cf0049e43c440ebe431545ed689633, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75a0d4a2b2be47e49ecee8de4f5bcdea, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5a64a6f1b0284c59a796117eeec61f0a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=108.7 K 2024-11-23T05:40:48,962 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e581eca25bfc4913bd6300c236ad078d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/395156a5568047e2bae6d10cd47ede6f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.2 K 2024-11-23T05:40:48,962 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:48,962 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/05cf0049e43c440ebe431545ed689633, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75a0d4a2b2be47e49ecee8de4f5bcdea, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5a64a6f1b0284c59a796117eeec61f0a] 2024-11-23T05:40:48,963 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e581eca25bfc4913bd6300c236ad078d, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340445595 2024-11-23T05:40:48,963 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 05cf0049e43c440ebe431545ed689633, keycount=150, bloomtype=ROW, size=30.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340445595 2024-11-23T05:40:48,963 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 395156a5568047e2bae6d10cd47ede6f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732340446277 2024-11-23T05:40:48,963 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 75a0d4a2b2be47e49ecee8de4f5bcdea, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732340446277 2024-11-23T05:40:48,964 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a8a73f5d4c0c4a47bcfa3e7a72f4a0c8, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340447011 2024-11-23T05:40:48,964 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5a64a6f1b0284c59a796117eeec61f0a, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340447011 2024-11-23T05:40:48,983 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:48,986 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#165 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:48,987 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a7165a6c7845495d94e1931b08640894 is 50, key is test_row_0/B:col10/1732340448458/Put/seqid=0 2024-11-23T05:40:48,996 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123891b1c80781342399e179ffe76c61bbb_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:48,999 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123891b1c80781342399e179ffe76c61bbb_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:48,999 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123891b1c80781342399e179ffe76c61bbb_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:49,008 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742022_1198 (size=12915) 2024-11-23T05:40:49,016 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a7165a6c7845495d94e1931b08640894 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a7165a6c7845495d94e1931b08640894 2024-11-23T05:40:49,026 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into a7165a6c7845495d94e1931b08640894(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:49,026 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:49,026 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=13, startTime=1732340448960; duration=0sec 2024-11-23T05:40:49,026 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:49,027 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:49,027 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:49,028 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37115 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:49,029 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:49,029 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:49,029 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3184060ed6a34116a55f7570bb642ea9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b3da6d269e234959821ab56114edf2e0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffba094c8f6348fc8747c3c19d157249] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.2 K 2024-11-23T05:40:49,030 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3184060ed6a34116a55f7570bb642ea9, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340445595 2024-11-23T05:40:49,030 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b3da6d269e234959821ab56114edf2e0, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732340446277 2024-11-23T05:40:49,031 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ffba094c8f6348fc8747c3c19d157249, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340447011 2024-11-23T05:40:49,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742023_1199 (size=4469) 2024-11-23T05:40:49,040 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#164 average throughput is 0.43 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:49,040 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/d3e77694277747dc9ab88e3f7a17f86b is 175, key is test_row_0/A:col10/1732340448458/Put/seqid=0 2024-11-23T05:40:49,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742024_1200 (size=31869) 2024-11-23T05:40:49,055 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#166 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:49,055 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/3e91d672462e443d83d4c0adb38bc6f7 is 50, key is test_row_0/C:col10/1732340448458/Put/seqid=0 2024-11-23T05:40:49,062 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/d3e77694277747dc9ab88e3f7a17f86b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/d3e77694277747dc9ab88e3f7a17f86b 2024-11-23T05:40:49,072 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=53 2024-11-23T05:40:49,072 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:49,072 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:40:49,075 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into d3e77694277747dc9ab88e3f7a17f86b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:49,076 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:49,076 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=13, startTime=1732340448960; duration=0sec 2024-11-23T05:40:49,076 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:49,076 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:49,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742025_1201 (size=12915) 2024-11-23T05:40:49,097 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232357ce8acf634989aa3f8b94607dad53_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340448482/Put/seqid=0 2024-11-23T05:40:49,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:49,122 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:49,124 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742026_1202 (size=12454) 2024-11-23T05:40:49,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:49,131 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232357ce8acf634989aa3f8b94607dad53_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232357ce8acf634989aa3f8b94607dad53_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:49,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/1109d03a77d94d68b2c697d335dfdf8c, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:49,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/1109d03a77d94d68b2c697d335dfdf8c is 175, key is test_row_0/A:col10/1732340448482/Put/seqid=0 2024-11-23T05:40:49,144 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742027_1203 (size=31255) 2024-11-23T05:40:49,145 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=291, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/1109d03a77d94d68b2c697d335dfdf8c 2024-11-23T05:40:49,152 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a1da071df45c42949a17ec62d7f3e14a is 50, key is test_row_0/B:col10/1732340448482/Put/seqid=0 2024-11-23T05:40:49,166 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742028_1204 (size=12301) 2024-11-23T05:40:49,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340509170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,173 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340509170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,173 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,174 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,178 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340509176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,179 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340509176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,179 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:49,179 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,180 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340509186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,189 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,189 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,277 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340509276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340509276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,278 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340509486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340509486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,491 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,493 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/3e91d672462e443d83d4c0adb38bc6f7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3e91d672462e443d83d4c0adb38bc6f7 2024-11-23T05:40:49,499 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into 3e91d672462e443d83d4c0adb38bc6f7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:49,499 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:49,499 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=13, startTime=1732340448960; duration=0sec 2024-11-23T05:40:49,499 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:49,499 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:49,567 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a1da071df45c42949a17ec62d7f3e14a 2024-11-23T05:40:49,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/403f3bf9169247dfb7be3d432c7f8e6a is 50, key is test_row_0/C:col10/1732340448482/Put/seqid=0 2024-11-23T05:40:49,579 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742029_1205 (size=12301) 2024-11-23T05:40:49,797 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340509795, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340509796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:49,798 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,015 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=291 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/403f3bf9169247dfb7be3d432c7f8e6a 2024-11-23T05:40:50,020 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/1109d03a77d94d68b2c697d335dfdf8c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/1109d03a77d94d68b2c697d335dfdf8c 2024-11-23T05:40:50,024 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/1109d03a77d94d68b2c697d335dfdf8c, entries=150, sequenceid=291, filesize=30.5 K 2024-11-23T05:40:50,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/a1da071df45c42949a17ec62d7f3e14a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a1da071df45c42949a17ec62d7f3e14a 2024-11-23T05:40:50,029 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a1da071df45c42949a17ec62d7f3e14a, entries=150, sequenceid=291, filesize=12.0 K 2024-11-23T05:40:50,030 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/403f3bf9169247dfb7be3d432c7f8e6a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/403f3bf9169247dfb7be3d432c7f8e6a 2024-11-23T05:40:50,036 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/403f3bf9169247dfb7be3d432c7f8e6a, entries=150, sequenceid=291, filesize=12.0 K 2024-11-23T05:40:50,037 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 6fed8128b8c4dd3776e5696d683dbd53 in 965ms, sequenceid=291, compaction requested=false 2024-11-23T05:40:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=53}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=53 2024-11-23T05:40:50,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=53 2024-11-23T05:40:50,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=53, resume processing ppid=52 2024-11-23T05:40:50,040 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=53, ppid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.8900 sec 2024-11-23T05:40:50,041 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=52, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=52, table=TestAcidGuarantees in 3.8940 sec 2024-11-23T05:40:50,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=52 2024-11-23T05:40:50,307 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:50,307 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:50,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees 2024-11-23T05:40:50,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T05:40:50,309 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:50,309 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=54, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:50,309 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=55, ppid=54, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:50,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:50,311 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:50,311 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:40:50,322 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232b86c960cc264af890be84c90c228981_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340450309/Put/seqid=0 2024-11-23T05:40:50,329 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742030_1206 (size=12454) 2024-11-23T05:40:50,332 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340510328, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,333 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,334 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340510333, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,334 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T05:40:50,439 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340510436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,440 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,447 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 109 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340510446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,448 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,466 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,466 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,466 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,619 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,619 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,619 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T05:40:50,646 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 117 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340510646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,647 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,657 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 110 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340510656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,657 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,658 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,730 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:50,734 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232b86c960cc264af890be84c90c228981_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232b86c960cc264af890be84c90c228981_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:50,735 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/0fc8d507b4894d07a145769036deaecd, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:50,735 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/0fc8d507b4894d07a145769036deaecd is 175, key is test_row_0/A:col10/1732340450309/Put/seqid=0 2024-11-23T05:40:50,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742031_1207 (size=31255) 2024-11-23T05:40:50,771 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:50,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,772 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,772 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,926 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:50,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:50,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:50,927 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,927 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:50,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T05:40:50,959 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 118 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340510956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,959 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:50,970 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:50,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 111 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340510966, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:50,970 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:50,971 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:51,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:51,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:51,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,080 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,139 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=317, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/0fc8d507b4894d07a145769036deaecd 2024-11-23T05:40:51,146 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/3bad8b20e86b4b8e8493b009e241a034 is 50, key is test_row_0/B:col10/1732340450309/Put/seqid=0 2024-11-23T05:40:51,149 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742032_1208 (size=12301) 2024-11-23T05:40:51,196 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:51,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340511196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:51,196 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,197 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:51,206 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:51,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340511206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:51,206 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:51,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340511206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:51,207 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:51,207 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:51,207 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:51,232 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:51,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:51,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,233 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,233 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,385 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:51,385 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,386 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,386 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T05:40:51,468 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:51,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 119 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340511466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,469 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:51,478 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:51,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 112 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340511476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:51,478 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:51,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:51,538 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:51,538 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:51,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,539 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,539 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,550 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/3bad8b20e86b4b8e8493b009e241a034 2024-11-23T05:40:51,556 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/5f86470834d74182b836f5e1907ed019 is 50, key is test_row_0/C:col10/1732340450309/Put/seqid=0 2024-11-23T05:40:51,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742033_1209 (size=12301) 2024-11-23T05:40:51,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] handler.RSProcedureHandler(58): pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=55 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=55 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:51,960 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/5f86470834d74182b836f5e1907ed019 2024-11-23T05:40:51,964 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/0fc8d507b4894d07a145769036deaecd as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0fc8d507b4894d07a145769036deaecd 2024-11-23T05:40:51,969 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0fc8d507b4894d07a145769036deaecd, entries=150, sequenceid=317, filesize=30.5 K 2024-11-23T05:40:51,970 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/3bad8b20e86b4b8e8493b009e241a034 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3bad8b20e86b4b8e8493b009e241a034 2024-11-23T05:40:51,974 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3bad8b20e86b4b8e8493b009e241a034, entries=150, sequenceid=317, filesize=12.0 K 2024-11-23T05:40:51,975 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/5f86470834d74182b836f5e1907ed019 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/5f86470834d74182b836f5e1907ed019 2024-11-23T05:40:51,981 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/5f86470834d74182b836f5e1907ed019, entries=150, sequenceid=317, filesize=12.0 K 2024-11-23T05:40:51,982 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 6fed8128b8c4dd3776e5696d683dbd53 in 1671ms, sequenceid=317, compaction requested=true 2024-11-23T05:40:51,982 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:51,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:51,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:51,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:51,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:51,982 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:51,982 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:51,983 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T05:40:51,983 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:51,984 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:51,984 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 94379 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:51,984 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:51,984 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:51,985 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,985 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,985 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a7165a6c7845495d94e1931b08640894, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a1da071df45c42949a17ec62d7f3e14a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3bad8b20e86b4b8e8493b009e241a034] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.6 K 2024-11-23T05:40:51,985 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/d3e77694277747dc9ab88e3f7a17f86b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/1109d03a77d94d68b2c697d335dfdf8c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0fc8d507b4894d07a145769036deaecd] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=92.2 K 2024-11-23T05:40:51,985 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,985 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/d3e77694277747dc9ab88e3f7a17f86b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/1109d03a77d94d68b2c697d335dfdf8c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0fc8d507b4894d07a145769036deaecd] 2024-11-23T05:40:51,985 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a7165a6c7845495d94e1931b08640894, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340447011 2024-11-23T05:40:51,985 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d3e77694277747dc9ab88e3f7a17f86b, keycount=150, bloomtype=ROW, size=31.1 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340447011 2024-11-23T05:40:51,986 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1109d03a77d94d68b2c697d335dfdf8c, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732340448469 2024-11-23T05:40:51,986 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a1da071df45c42949a17ec62d7f3e14a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732340448469 2024-11-23T05:40:51,986 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0fc8d507b4894d07a145769036deaecd, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340449161 2024-11-23T05:40:51,986 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3bad8b20e86b4b8e8493b009e241a034, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340449161 2024-11-23T05:40:51,996 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:51,998 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=55 2024-11-23T05:40:51,998 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#173 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:51,998 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:51,998 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:40:51,998 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c5f2ffc514af433294a6caa89edabb43 is 50, key is test_row_0/B:col10/1732340450309/Put/seqid=0 2024-11-23T05:40:52,007 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123f6a16e0a3c654523980ffa92ff03fc80_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:52,009 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123f6a16e0a3c654523980ffa92ff03fc80_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:52,010 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f6a16e0a3c654523980ffa92ff03fc80_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:52,013 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232f9dc8e8c3094fb497e7f29cce3dffd1_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340450319/Put/seqid=0 2024-11-23T05:40:52,018 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742034_1210 (size=13017) 2024-11-23T05:40:52,028 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c5f2ffc514af433294a6caa89edabb43 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c5f2ffc514af433294a6caa89edabb43 2024-11-23T05:40:52,033 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into c5f2ffc514af433294a6caa89edabb43(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:52,033 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:52,033 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=13, startTime=1732340451982; duration=0sec 2024-11-23T05:40:52,033 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:52,033 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:52,033 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:52,035 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37517 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:52,035 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:52,035 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,035 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3e91d672462e443d83d4c0adb38bc6f7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/403f3bf9169247dfb7be3d432c7f8e6a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/5f86470834d74182b836f5e1907ed019] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.6 K 2024-11-23T05:40:52,036 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3e91d672462e443d83d4c0adb38bc6f7, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340447011 2024-11-23T05:40:52,036 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 403f3bf9169247dfb7be3d432c7f8e6a, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=291, earliestPutTs=1732340448469 2024-11-23T05:40:52,037 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5f86470834d74182b836f5e1907ed019, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340449161 2024-11-23T05:40:52,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742035_1211 (size=4469) 2024-11-23T05:40:52,041 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#174 average throughput is 0.54 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:52,041 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/f394a2a56a6a438da61760f28af555e9 is 175, key is test_row_0/A:col10/1732340450309/Put/seqid=0 2024-11-23T05:40:52,048 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742036_1212 (size=12454) 2024-11-23T05:40:52,052 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#176 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:52,053 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/f27d851276164ab88324b6d07ac6f00e is 50, key is test_row_0/C:col10/1732340450309/Put/seqid=0 2024-11-23T05:40:52,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,059 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411232f9dc8e8c3094fb497e7f29cce3dffd1_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232f9dc8e8c3094fb497e7f29cce3dffd1_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:52,060 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9eec56823df041b8a4b27f4ed2c433b7, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:52,061 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742037_1213 (size=31971) 2024-11-23T05:40:52,061 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9eec56823df041b8a4b27f4ed2c433b7 is 175, key is test_row_0/A:col10/1732340450319/Put/seqid=0 2024-11-23T05:40:52,067 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/f394a2a56a6a438da61760f28af555e9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/f394a2a56a6a438da61760f28af555e9 2024-11-23T05:40:52,075 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into f394a2a56a6a438da61760f28af555e9(size=31.2 K), total size for store is 31.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:52,075 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:52,075 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=13, startTime=1732340451982; duration=0sec 2024-11-23T05:40:52,075 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:52,075 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:52,080 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742039_1215 (size=31255) 2024-11-23T05:40:52,081 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=329, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9eec56823df041b8a4b27f4ed2c433b7 2024-11-23T05:40:52,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/0343a22142764e84855668aca24e3d03 is 50, key is test_row_0/B:col10/1732340450319/Put/seqid=0 2024-11-23T05:40:52,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742038_1214 (size=13017) 2024-11-23T05:40:52,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742040_1216 (size=12301) 2024-11-23T05:40:52,103 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/0343a22142764e84855668aca24e3d03 2024-11-23T05:40:52,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/c5bae7caf77e4c00ac52293be1831200 is 50, key is test_row_0/C:col10/1732340450319/Put/seqid=0 2024-11-23T05:40:52,126 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742041_1217 (size=12301) 2024-11-23T05:40:52,127 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=329 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/c5bae7caf77e4c00ac52293be1831200 2024-11-23T05:40:52,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9eec56823df041b8a4b27f4ed2c433b7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9eec56823df041b8a4b27f4ed2c433b7 2024-11-23T05:40:52,136 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9eec56823df041b8a4b27f4ed2c433b7, entries=150, sequenceid=329, filesize=30.5 K 2024-11-23T05:40:52,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/0343a22142764e84855668aca24e3d03 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0343a22142764e84855668aca24e3d03 2024-11-23T05:40:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,142 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0343a22142764e84855668aca24e3d03, entries=150, sequenceid=329, filesize=12.0 K 2024-11-23T05:40:52,142 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/c5bae7caf77e4c00ac52293be1831200 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/c5bae7caf77e4c00ac52293be1831200 2024-11-23T05:40:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,147 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/c5bae7caf77e4c00ac52293be1831200, entries=150, sequenceid=329, filesize=12.0 K 2024-11-23T05:40:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,148 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=0 B/0 for 6fed8128b8c4dd3776e5696d683dbd53 in 150ms, sequenceid=329, compaction requested=false 2024-11-23T05:40:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=55}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=55 2024-11-23T05:40:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=55 2024-11-23T05:40:52,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,165 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=55, resume processing ppid=54 2024-11-23T05:40:52,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,166 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=55, ppid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8410 sec 2024-11-23T05:40:52,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,168 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=54, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=54, table=TestAcidGuarantees in 1.8600 sec 2024-11-23T05:40:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,171 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,172 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,176 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,178 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,180 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,181 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,186 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,191 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,195 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,196 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,199 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,200 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,207 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,209 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,213 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,214 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,215 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,220 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,221 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,224 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,225 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,229 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,231 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,234 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,239 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,240 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,241 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,244 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,245 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,250 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,255 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,258 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,263 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,264 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,268 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,274 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,280 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,281 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,283 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,284 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,289 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,290 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,292 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,293 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,295 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,297 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,301 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,302 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,304 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,305 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,307 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,310 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,312 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,314 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,321 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,322 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,323 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,325 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,327 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,329 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,330 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,331 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,332 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,333 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,334 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,335 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,342 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,343 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,344 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,345 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,346 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T05:40:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,348 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,349 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,352 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,353 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,355 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,360 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,371 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,372 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,373 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,374 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,375 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,377 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,378 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,380 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,381 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,382 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,383 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,384 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,389 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,394 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,400 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,401 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,403 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,407 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,408 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,413 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,414 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,417 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,444 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,445 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=54 2024-11-23T05:40:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,457 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,458 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees 2024-11-23T05:40:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,460 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,461 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=56, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,461 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=57, ppid=56, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:52,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T05:40:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,472 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,503 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/f27d851276164ab88324b6d07ac6f00e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/f27d851276164ab88324b6d07ac6f00e 2024-11-23T05:40:52,503 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,511 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into f27d851276164ab88324b6d07ac6f00e(size=12.7 K), total size for store is 24.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:52,511 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:52,511 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=13, startTime=1732340451982; duration=0sec 2024-11-23T05:40:52,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,511 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,512 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T05:40:52,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:52,580 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:52,580 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:52,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,616 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:52,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:52,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,616 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,617 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,622 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ad7ac8f709d941b2a99c9f7d771b79ef_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340452578/Put/seqid=0 2024-11-23T05:40:52,640 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742043_1219 (size=25158) 2024-11-23T05:40:52,642 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:52,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ad7ac8f709d941b2a99c9f7d771b79ef_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ad7ac8f709d941b2a99c9f7d771b79ef_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:52,653 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/91ff9f0e389144e2b190042b3bb39499, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:52,655 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/91ff9f0e389144e2b190042b3bb39499 is 175, key is test_row_0/A:col10/1732340452578/Put/seqid=0 2024-11-23T05:40:52,666 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742042_1218 (size=74795) 2024-11-23T05:40:52,685 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:52,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340512684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,686 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:52,691 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:52,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 128 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340512687, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,692 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:52,770 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:52,770 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,771 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,771 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T05:40:52,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:52,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340512798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:52,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,800 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:52,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:52,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 129 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340512798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:52,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:52,800 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:52,924 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:52,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,924 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:52,925 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,925 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:52,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 138 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340513006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,009 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:53,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 130 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340513006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,010 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:53,068 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=343, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/91ff9f0e389144e2b190042b3bb39499 2024-11-23T05:40:53,078 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:53,078 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,079 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,079 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,087 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T05:40:53,096 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/e501268c9f7e49ddb95481df0902921b is 50, key is test_row_0/B:col10/1732340452578/Put/seqid=0 2024-11-23T05:40:53,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742044_1220 (size=12301) 2024-11-23T05:40:53,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:53,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:53,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,235 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:53,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 139 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340513316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:53,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:53,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 131 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340513316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,317 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:53,389 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,389 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,390 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,508 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/e501268c9f7e49ddb95481df0902921b 2024-11-23T05:40:53,530 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/7527290d06414ac9a7dbc7a82bb81446 is 50, key is test_row_0/C:col10/1732340452578/Put/seqid=0 2024-11-23T05:40:53,542 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:53,542 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,542 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,544 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742045_1221 (size=12301) 2024-11-23T05:40:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T05:40:53,696 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,696 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,696 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,826 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 140 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340513826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,827 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:53,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 132 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340513826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:53,827 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:53,849 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:53,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:53,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,849 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] handler.RSProcedureHandler(58): pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,849 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=57 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=57 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:53,945 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=343 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/7527290d06414ac9a7dbc7a82bb81446 2024-11-23T05:40:53,950 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/91ff9f0e389144e2b190042b3bb39499 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/91ff9f0e389144e2b190042b3bb39499 2024-11-23T05:40:53,954 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/91ff9f0e389144e2b190042b3bb39499, entries=400, sequenceid=343, filesize=73.0 K 2024-11-23T05:40:53,954 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/e501268c9f7e49ddb95481df0902921b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e501268c9f7e49ddb95481df0902921b 2024-11-23T05:40:53,959 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e501268c9f7e49ddb95481df0902921b, entries=150, sequenceid=343, filesize=12.0 K 2024-11-23T05:40:53,960 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/7527290d06414ac9a7dbc7a82bb81446 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/7527290d06414ac9a7dbc7a82bb81446 2024-11-23T05:40:53,964 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/7527290d06414ac9a7dbc7a82bb81446, entries=150, sequenceid=343, filesize=12.0 K 2024-11-23T05:40:53,965 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6fed8128b8c4dd3776e5696d683dbd53 in 1385ms, sequenceid=343, compaction requested=true 2024-11-23T05:40:53,965 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:53,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:53,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:53,965 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:53,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:53,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:53,965 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:53,966 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T05:40:53,966 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:53,966 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 138021 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:53,967 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:53,967 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,967 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/f394a2a56a6a438da61760f28af555e9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9eec56823df041b8a4b27f4ed2c433b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/91ff9f0e389144e2b190042b3bb39499] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=134.8 K 2024-11-23T05:40:53,967 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,967 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/f394a2a56a6a438da61760f28af555e9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9eec56823df041b8a4b27f4ed2c433b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/91ff9f0e389144e2b190042b3bb39499] 2024-11-23T05:40:53,967 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:53,967 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f394a2a56a6a438da61760f28af555e9, keycount=150, bloomtype=ROW, size=31.2 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340449161 2024-11-23T05:40:53,967 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:53,967 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:53,967 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c5f2ffc514af433294a6caa89edabb43, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0343a22142764e84855668aca24e3d03, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e501268c9f7e49ddb95481df0902921b] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.7 K 2024-11-23T05:40:53,968 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9eec56823df041b8a4b27f4ed2c433b7, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732340450319 2024-11-23T05:40:53,968 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 91ff9f0e389144e2b190042b3bb39499, keycount=400, bloomtype=ROW, size=73.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732340452493 2024-11-23T05:40:53,968 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting c5f2ffc514af433294a6caa89edabb43, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340449161 2024-11-23T05:40:53,968 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0343a22142764e84855668aca24e3d03, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732340450319 2024-11-23T05:40:53,968 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e501268c9f7e49ddb95481df0902921b, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732340452557 2024-11-23T05:40:53,981 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#182 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:53,981 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/3cde67bf853d4616b939f062f626620c is 50, key is test_row_0/B:col10/1732340452578/Put/seqid=0 2024-11-23T05:40:53,983 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:53,991 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123571f066600e94ec6946a934e7917ac9b_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:53,993 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742046_1222 (size=13119) 2024-11-23T05:40:53,995 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123571f066600e94ec6946a934e7917ac9b_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:53,995 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123571f066600e94ec6946a934e7917ac9b_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:54,002 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=57 2024-11-23T05:40:54,002 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:54,002 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:40:54,005 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/3cde67bf853d4616b939f062f626620c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3cde67bf853d4616b939f062f626620c 2024-11-23T05:40:54,011 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into 3cde67bf853d4616b939f062f626620c(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:54,011 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:54,011 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=13, startTime=1732340453965; duration=0sec 2024-11-23T05:40:54,011 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:54,011 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:54,011 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:54,012 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37619 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:54,012 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:54,012 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:54,013 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/f27d851276164ab88324b6d07ac6f00e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/c5bae7caf77e4c00ac52293be1831200, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/7527290d06414ac9a7dbc7a82bb81446] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.7 K 2024-11-23T05:40:54,013 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting f27d851276164ab88324b6d07ac6f00e, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340449161 2024-11-23T05:40:54,013 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting c5bae7caf77e4c00ac52293be1831200, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=329, earliestPutTs=1732340450319 2024-11-23T05:40:54,013 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7527290d06414ac9a7dbc7a82bb81446, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732340452557 2024-11-23T05:40:54,020 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742047_1223 (size=4469) 2024-11-23T05:40:54,021 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#183 average throughput is 0.64 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:54,021 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/0bc684943b314ecdb1bedf88d8e080fe is 175, key is test_row_0/A:col10/1732340452578/Put/seqid=0 2024-11-23T05:40:54,031 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#184 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:54,031 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/89cab5dd1d494836bf9f0568eb508cef is 50, key is test_row_0/C:col10/1732340452578/Put/seqid=0 2024-11-23T05:40:54,034 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dee5732d95db4c65a9b435ca18ee5a97_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340452680/Put/seqid=0 2024-11-23T05:40:54,039 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742048_1224 (size=32073) 2024-11-23T05:40:54,051 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/0bc684943b314ecdb1bedf88d8e080fe as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0bc684943b314ecdb1bedf88d8e080fe 2024-11-23T05:40:54,059 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into 0bc684943b314ecdb1bedf88d8e080fe(size=31.3 K), total size for store is 31.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:54,059 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:54,059 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=13, startTime=1732340453965; duration=0sec 2024-11-23T05:40:54,060 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:54,060 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:54,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742049_1225 (size=13119) 2024-11-23T05:40:54,095 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/89cab5dd1d494836bf9f0568eb508cef as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/89cab5dd1d494836bf9f0568eb508cef 2024-11-23T05:40:54,103 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into 89cab5dd1d494836bf9f0568eb508cef(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:54,103 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742050_1226 (size=12454) 2024-11-23T05:40:54,103 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:54,103 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=13, startTime=1732340453965; duration=0sec 2024-11-23T05:40:54,103 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:54,103 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:54,105 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,113 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dee5732d95db4c65a9b435ca18ee5a97_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dee5732d95db4c65a9b435ca18ee5a97_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:54,114 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/4f5787a5d38a4175bdfbbcb067b68a6b, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:54,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/4f5787a5d38a4175bdfbbcb067b68a6b is 175, key is test_row_0/A:col10/1732340452680/Put/seqid=0 2024-11-23T05:40:54,122 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742051_1227 (size=31255) 2024-11-23T05:40:54,122 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=369, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/4f5787a5d38a4175bdfbbcb067b68a6b 2024-11-23T05:40:54,134 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/27b7b11c1b9d4975987cd621cd70522c is 50, key is test_row_0/B:col10/1732340452680/Put/seqid=0 2024-11-23T05:40:54,155 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742052_1228 (size=12301) 2024-11-23T05:40:54,554 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/27b7b11c1b9d4975987cd621cd70522c 2024-11-23T05:40:54,563 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/10a68370460948fdb7cdb225ad66d993 is 50, key is test_row_0/C:col10/1732340452680/Put/seqid=0 2024-11-23T05:40:54,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742053_1229 (size=12301) 2024-11-23T05:40:54,569 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=369 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/10a68370460948fdb7cdb225ad66d993 2024-11-23T05:40:54,575 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/4f5787a5d38a4175bdfbbcb067b68a6b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4f5787a5d38a4175bdfbbcb067b68a6b 2024-11-23T05:40:54,580 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4f5787a5d38a4175bdfbbcb067b68a6b, entries=150, sequenceid=369, filesize=30.5 K 2024-11-23T05:40:54,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/27b7b11c1b9d4975987cd621cd70522c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/27b7b11c1b9d4975987cd621cd70522c 2024-11-23T05:40:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,586 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/27b7b11c1b9d4975987cd621cd70522c, entries=150, sequenceid=369, filesize=12.0 K 2024-11-23T05:40:54,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/10a68370460948fdb7cdb225ad66d993 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/10a68370460948fdb7cdb225ad66d993 2024-11-23T05:40:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,592 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/10a68370460948fdb7cdb225ad66d993, entries=150, sequenceid=369, filesize=12.0 K 2024-11-23T05:40:54,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,593 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=0 B/0 for 6fed8128b8c4dd3776e5696d683dbd53 in 591ms, sequenceid=369, compaction requested=false 2024-11-23T05:40:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:54,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=57}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=57 2024-11-23T05:40:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=57 2024-11-23T05:40:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=57, resume processing ppid=56 2024-11-23T05:40:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,598 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=57, ppid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.1340 sec 2024-11-23T05:40:54,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,601 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=56, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=56, table=TestAcidGuarantees in 2.1410 sec 2024-11-23T05:40:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,615 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=56 2024-11-23T05:40:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,617 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,619 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees 2024-11-23T05:40:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T05:40:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,621 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=58, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,621 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=59, ppid=58, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:54,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,641 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,710 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T05:40:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,757 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,773 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=59 2024-11-23T05:40:54,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:54,774 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=59}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=59 2024-11-23T05:40:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=59 2024-11-23T05:40:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=59, resume processing ppid=58 2024-11-23T05:40:54,777 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=59, ppid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 154 msec 2024-11-23T05:40:54,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=58, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=58, table=TestAcidGuarantees in 159 msec 2024-11-23T05:40:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,844 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,878 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:54,924 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:40:54,925 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:40:54,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,935 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123abf7c20e768340e69957da1e0a1a8286_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340454915/Put/seqid=0 2024-11-23T05:40:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=58 2024-11-23T05:40:54,937 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:40:54,937 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:40:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees 2024-11-23T05:40:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T05:40:54,939 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:40:54,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,940 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=60, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:40:54,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=61, ppid=60, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:40:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:54,949 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742054_1230 (size=22618) 2024-11-23T05:40:55,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T05:40:55,048 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 148 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340515048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,049 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 157 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340515048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,051 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,095 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,095 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 158 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340515156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,159 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,159 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 149 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340515156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,159 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,160 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,208 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43092 deadline: 1732340515206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,209 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8194 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,209 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,219 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,219 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43052 deadline: 1732340515216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,219 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8213 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,220 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,247 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,248 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43080 deadline: 1732340515246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,249 WARN [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in 6fed8128b8c4dd3776e5696d683dbd53 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8235 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:40:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,249 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T05:40:55,349 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:55,358 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123abf7c20e768340e69957da1e0a1a8286_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123abf7c20e768340e69957da1e0a1a8286_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:55,359 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/c0d0ad36f17e41ec9d5bfec2f250abb1, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:55,359 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/c0d0ad36f17e41ec9d5bfec2f250abb1 is 175, key is test_row_0/A:col10/1732340454915/Put/seqid=0 2024-11-23T05:40:55,363 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742055_1231 (size=66023) 2024-11-23T05:40:55,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,367 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 159 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340515366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,367 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,368 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 150 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340515366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,368 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,401 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,401 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,401 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,402 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,451 INFO [Thread-646 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-646. 2024-11-23T05:40:55,451 DEBUG [Thread-646 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:55,451 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,451 DEBUG [Thread-646 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,451 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:55,452 INFO [Thread-644 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-644. 2024-11-23T05:40:55,452 DEBUG [Thread-644 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:55,452 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,452 DEBUG [Thread-644 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,452 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:55,455 INFO [Thread-650 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-650. 2024-11-23T05:40:55,455 DEBUG [Thread-650 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:55,456 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,456 DEBUG [Thread-650 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,456 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:55,461 INFO [Thread-648 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-648. 2024-11-23T05:40:55,461 DEBUG [Thread-648 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:55,461 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,461 DEBUG [Thread-648 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:55,461 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:55,555 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,555 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:55,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,556 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,556 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T05:40:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 160 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340515677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340515677, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,679 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:55,712 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:55,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,712 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,764 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=384, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/c0d0ad36f17e41ec9d5bfec2f250abb1 2024-11-23T05:40:55,777 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c18cf0f5ef6d45f8ac45b60c78fe11d1 is 50, key is test_row_0/B:col10/1732340454915/Put/seqid=0 2024-11-23T05:40:55,781 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742056_1232 (size=12301) 2024-11-23T05:40:55,869 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:55,869 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:55,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,023 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:56,023 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,024 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,077 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T05:40:56,180 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:56,180 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,181 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,182 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c18cf0f5ef6d45f8ac45b60c78fe11d1 2024-11-23T05:40:56,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,186 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:56,187 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:40:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 161 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43082 deadline: 1732340516186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:56,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 152 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:43056 deadline: 1732340516186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:40:56,187 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:56,187 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:40:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:56,188 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:40:56,192 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/613ee014d56440c58359713aecaa2d79 is 50, key is test_row_0/C:col10/1732340454915/Put/seqid=0 2024-11-23T05:40:56,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742057_1233 (size=12301) 2024-11-23T05:40:56,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:56,336 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,493 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:56,494 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,494 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] handler.RSProcedureHandler(58): pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,495 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=61 java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=61 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:40:56,597 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=384 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/613ee014d56440c58359713aecaa2d79 2024-11-23T05:40:56,602 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/c0d0ad36f17e41ec9d5bfec2f250abb1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c0d0ad36f17e41ec9d5bfec2f250abb1 2024-11-23T05:40:56,607 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c0d0ad36f17e41ec9d5bfec2f250abb1, entries=350, sequenceid=384, filesize=64.5 K 2024-11-23T05:40:56,608 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/c18cf0f5ef6d45f8ac45b60c78fe11d1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c18cf0f5ef6d45f8ac45b60c78fe11d1 2024-11-23T05:40:56,612 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c18cf0f5ef6d45f8ac45b60c78fe11d1, entries=150, sequenceid=384, filesize=12.0 K 2024-11-23T05:40:56,613 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/613ee014d56440c58359713aecaa2d79 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/613ee014d56440c58359713aecaa2d79 2024-11-23T05:40:56,617 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/613ee014d56440c58359713aecaa2d79, entries=150, sequenceid=384, filesize=12.0 K 2024-11-23T05:40:56,618 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=154.31 KB/158010 for 6fed8128b8c4dd3776e5696d683dbd53 in 1694ms, sequenceid=384, compaction requested=true 2024-11-23T05:40:56,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:56,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:40:56,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:56,618 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:56,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:40:56,618 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:56,619 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 6fed8128b8c4dd3776e5696d683dbd53:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:40:56,619 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:56,619 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:56,619 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 129351 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:56,620 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/B is initiating minor compaction (all files) 2024-11-23T05:40:56,620 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/A is initiating minor compaction (all files) 2024-11-23T05:40:56,620 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/B in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,620 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/A in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,620 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3cde67bf853d4616b939f062f626620c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/27b7b11c1b9d4975987cd621cd70522c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c18cf0f5ef6d45f8ac45b60c78fe11d1] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.8 K 2024-11-23T05:40:56,620 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0bc684943b314ecdb1bedf88d8e080fe, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4f5787a5d38a4175bdfbbcb067b68a6b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c0d0ad36f17e41ec9d5bfec2f250abb1] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=126.3 K 2024-11-23T05:40:56,620 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,620 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0bc684943b314ecdb1bedf88d8e080fe, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4f5787a5d38a4175bdfbbcb067b68a6b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c0d0ad36f17e41ec9d5bfec2f250abb1] 2024-11-23T05:40:56,620 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3cde67bf853d4616b939f062f626620c, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732340452557 2024-11-23T05:40:56,620 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0bc684943b314ecdb1bedf88d8e080fe, keycount=150, bloomtype=ROW, size=31.3 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732340452557 2024-11-23T05:40:56,620 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 27b7b11c1b9d4975987cd621cd70522c, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732340452666 2024-11-23T05:40:56,621 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4f5787a5d38a4175bdfbbcb067b68a6b, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732340452666 2024-11-23T05:40:56,621 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting c18cf0f5ef6d45f8ac45b60c78fe11d1, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732340454915 2024-11-23T05:40:56,621 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c0d0ad36f17e41ec9d5bfec2f250abb1, keycount=350, bloomtype=ROW, size=64.5 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732340454870 2024-11-23T05:40:56,630 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#B#compaction#191 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:56,630 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/df96e845502f41d7be0af56edc81794d is 50, key is test_row_0/B:col10/1732340454915/Put/seqid=0 2024-11-23T05:40:56,631 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:56,641 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123995ed3945879467896245af27df2f2a1_6fed8128b8c4dd3776e5696d683dbd53 store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:56,642 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742058_1234 (size=13221) 2024-11-23T05:40:56,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=61 2024-11-23T05:40:56,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,650 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:40:56,683 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c63fd9fadc7b4065a87b0e362fc51852_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340455027/Put/seqid=0 2024-11-23T05:40:56,689 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123995ed3945879467896245af27df2f2a1_6fed8128b8c4dd3776e5696d683dbd53, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:56,690 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123995ed3945879467896245af27df2f2a1_6fed8128b8c4dd3776e5696d683dbd53 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:56,690 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742059_1235 (size=12454) 2024-11-23T05:40:56,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:40:56,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742060_1236 (size=4469) 2024-11-23T05:40:56,698 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#A#compaction#192 average throughput is 0.36 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:56,699 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/4d6918587cb044d08a06137eb4bec1b3 is 175, key is test_row_0/A:col10/1732340454915/Put/seqid=0 2024-11-23T05:40:56,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742061_1237 (size=32175) 2024-11-23T05:40:56,705 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123c63fd9fadc7b4065a87b0e362fc51852_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c63fd9fadc7b4065a87b0e362fc51852_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:56,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/15ce1a777a964c6ca01c0a6450a697d7, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:40:56,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/15ce1a777a964c6ca01c0a6450a697d7 is 175, key is test_row_0/A:col10/1732340455027/Put/seqid=0 2024-11-23T05:40:56,713 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/4d6918587cb044d08a06137eb4bec1b3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4d6918587cb044d08a06137eb4bec1b3 2024-11-23T05:40:56,716 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742062_1238 (size=31255) 2024-11-23T05:40:56,719 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/A of 6fed8128b8c4dd3776e5696d683dbd53 into 4d6918587cb044d08a06137eb4bec1b3(size=31.4 K), total size for store is 31.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:56,719 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:56,719 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/A, priority=13, startTime=1732340456618; duration=0sec 2024-11-23T05:40:56,719 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:40:56,719 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:A 2024-11-23T05:40:56,719 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:40:56,720 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37721 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:40:56,720 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 6fed8128b8c4dd3776e5696d683dbd53/C is initiating minor compaction (all files) 2024-11-23T05:40:56,721 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 6fed8128b8c4dd3776e5696d683dbd53/C in TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:56,721 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/89cab5dd1d494836bf9f0568eb508cef, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/10a68370460948fdb7cdb225ad66d993, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/613ee014d56440c58359713aecaa2d79] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp, totalSize=36.8 K 2024-11-23T05:40:56,721 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 89cab5dd1d494836bf9f0568eb508cef, keycount=150, bloomtype=ROW, size=12.8 K, encoding=NONE, compression=NONE, seqNum=343, earliestPutTs=1732340452557 2024-11-23T05:40:56,722 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 10a68370460948fdb7cdb225ad66d993, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=369, earliestPutTs=1732340452666 2024-11-23T05:40:56,722 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 613ee014d56440c58359713aecaa2d79, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=384, earliestPutTs=1732340454915 2024-11-23T05:40:56,733 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 6fed8128b8c4dd3776e5696d683dbd53#C#compaction#194 average throughput is 1.31 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:40:56,734 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/04c00c37a6ae4404bff89536363c82ad is 50, key is test_row_0/C:col10/1732340454915/Put/seqid=0 2024-11-23T05:40:56,738 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742063_1239 (size=13221) 2024-11-23T05:40:57,047 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/df96e845502f41d7be0af56edc81794d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/df96e845502f41d7be0af56edc81794d 2024-11-23T05:40:57,052 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/B of 6fed8128b8c4dd3776e5696d683dbd53 into df96e845502f41d7be0af56edc81794d(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:57,052 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:57,052 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/B, priority=13, startTime=1732340456618; duration=0sec 2024-11-23T05:40:57,052 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:57,053 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:B 2024-11-23T05:40:57,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T05:40:57,114 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=410, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/15ce1a777a964c6ca01c0a6450a697d7 2024-11-23T05:40:57,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/43914c6f1b9b4134bee2004afaaf1904 is 50, key is test_row_0/B:col10/1732340455027/Put/seqid=0 2024-11-23T05:40:57,137 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742064_1240 (size=12301) 2024-11-23T05:40:57,145 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/04c00c37a6ae4404bff89536363c82ad as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/04c00c37a6ae4404bff89536363c82ad 2024-11-23T05:40:57,150 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 6fed8128b8c4dd3776e5696d683dbd53/C of 6fed8128b8c4dd3776e5696d683dbd53 into 04c00c37a6ae4404bff89536363c82ad(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:40:57,150 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:57,150 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., storeName=6fed8128b8c4dd3776e5696d683dbd53/C, priority=13, startTime=1732340456618; duration=0sec 2024-11-23T05:40:57,150 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:40:57,150 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 6fed8128b8c4dd3776e5696d683dbd53:C 2024-11-23T05:40:57,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:40:57,197 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. as already flushing 2024-11-23T05:40:57,198 INFO [Thread-633 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-633. 2024-11-23T05:40:57,198 DEBUG [Thread-633 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:57,198 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:57,198 DEBUG [Thread-633 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:57,198 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:57,207 INFO [Thread-637 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-637. 2024-11-23T05:40:57,207 DEBUG [Thread-637 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:40:57,207 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:57,207 DEBUG [Thread-637 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:40:57,208 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:40:57,538 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/43914c6f1b9b4134bee2004afaaf1904 2024-11-23T05:40:57,548 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/987d9ea825854b4bb244c862ce361c74 is 50, key is test_row_0/C:col10/1732340455027/Put/seqid=0 2024-11-23T05:40:57,552 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742065_1241 (size=12301) 2024-11-23T05:40:57,953 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=410 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/987d9ea825854b4bb244c862ce361c74 2024-11-23T05:40:57,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/15ce1a777a964c6ca01c0a6450a697d7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/15ce1a777a964c6ca01c0a6450a697d7 2024-11-23T05:40:57,962 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/15ce1a777a964c6ca01c0a6450a697d7, entries=150, sequenceid=410, filesize=30.5 K 2024-11-23T05:40:57,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/43914c6f1b9b4134bee2004afaaf1904 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/43914c6f1b9b4134bee2004afaaf1904 2024-11-23T05:40:57,969 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/43914c6f1b9b4134bee2004afaaf1904, entries=150, sequenceid=410, filesize=12.0 K 2024-11-23T05:40:57,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/987d9ea825854b4bb244c862ce361c74 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/987d9ea825854b4bb244c862ce361c74 2024-11-23T05:40:57,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/987d9ea825854b4bb244c862ce361c74, entries=150, sequenceid=410, filesize=12.0 K 2024-11-23T05:40:57,975 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=13.42 KB/13740 for 6fed8128b8c4dd3776e5696d683dbd53 in 1325ms, sequenceid=410, compaction requested=false 2024-11-23T05:40:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.HRegion(2603): Flush status journal for 6fed8128b8c4dd3776e5696d683dbd53: 2024-11-23T05:40:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:40:57,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=61}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=61 2024-11-23T05:40:57,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=61 2024-11-23T05:40:57,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=61, resume processing ppid=60 2024-11-23T05:40:57,978 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=61, ppid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.0360 sec 2024-11-23T05:40:57,980 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=60, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=60, table=TestAcidGuarantees in 3.0420 sec 2024-11-23T05:40:59,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=60 2024-11-23T05:40:59,096 INFO [RPCClient-NioEventLoopGroup-4-7 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:05,267 INFO [Thread-641 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-641. 2024-11-23T05:41:05,268 DEBUG [Thread-641 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:05,268 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,268 DEBUG [Thread-641 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,268 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:05,288 INFO [Thread-639 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-639. 2024-11-23T05:41:05,288 DEBUG [Thread-639 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:05,288 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,288 DEBUG [Thread-639 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,288 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:05,307 INFO [Thread-635 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-635. 2024-11-23T05:41:05,307 DEBUG [Thread-635 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:05,307 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,307 DEBUG [Thread-635 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,307 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 97 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 88 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 48 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3367 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3346 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1479 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4437 rows 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1471 2024-11-23T05:41:05,307 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4413 rows 2024-11-23T05:41:05,307 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:41:05,308 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobMixedAtomicity(AcidGuaranteesTestBase.java:132) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:41:05,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,308 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:05,308 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:05,308 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T05:41:05,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=62, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:05,311 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T05:41:05,311 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340465311"}]},"ts":"1732340465311"} 2024-11-23T05:41:05,313 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T05:41:05,313 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T05:41:05,314 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=63, ppid=62, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:41:05,318 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, UNASSIGN}] 2024-11-23T05:41:05,319 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, UNASSIGN 2024-11-23T05:41:05,320 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:05,321 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=64, ppid=63, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, UNASSIGN because future has completed 2024-11-23T05:41:05,321 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:41:05,321 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:41:05,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T05:41:05,473 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(122): Close 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:05,473 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:41:05,474 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1722): Closing 6fed8128b8c4dd3776e5696d683dbd53, disabling compactions & flushes 2024-11-23T05:41:05,474 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:41:05,474 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:41:05,474 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. after waiting 0 ms 2024-11-23T05:41:05,474 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:41:05,474 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(2902): Flushing 6fed8128b8c4dd3776e5696d683dbd53 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T05:41:05,479 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231d67e382af97461089d1fc52318dbe8e_6fed8128b8c4dd3776e5696d683dbd53 is 50, key is test_row_0/A:col10/1732340465266/Put/seqid=0 2024-11-23T05:41:05,483 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742066_1242 (size=12454) 2024-11-23T05:41:05,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T05:41:05,884 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:05,889 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411231d67e382af97461089d1fc52318dbe8e_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231d67e382af97461089d1fc52318dbe8e_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:05,889 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9bc737f3439b4afc829ddc081fef0610, store: [table=TestAcidGuarantees family=A region=6fed8128b8c4dd3776e5696d683dbd53] 2024-11-23T05:41:05,890 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9bc737f3439b4afc829ddc081fef0610 is 175, key is test_row_0/A:col10/1732340465266/Put/seqid=0 2024-11-23T05:41:05,894 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742067_1243 (size=31255) 2024-11-23T05:41:05,895 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=421, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9bc737f3439b4afc829ddc081fef0610 2024-11-23T05:41:05,902 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/6a675e4687b94a25acb81f18acee0741 is 50, key is test_row_0/B:col10/1732340465266/Put/seqid=0 2024-11-23T05:41:05,905 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742068_1244 (size=12301) 2024-11-23T05:41:05,906 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/6a675e4687b94a25acb81f18acee0741 2024-11-23T05:41:05,916 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/affcf173c2434923a5efb6e90261f69e is 50, key is test_row_0/C:col10/1732340465266/Put/seqid=0 2024-11-23T05:41:05,924 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742069_1245 (size=12301) 2024-11-23T05:41:05,924 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=421 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/affcf173c2434923a5efb6e90261f69e 2024-11-23T05:41:05,930 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/A/9bc737f3439b4afc829ddc081fef0610 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9bc737f3439b4afc829ddc081fef0610 2024-11-23T05:41:05,935 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9bc737f3439b4afc829ddc081fef0610, entries=150, sequenceid=421, filesize=30.5 K 2024-11-23T05:41:05,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T05:41:05,936 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/B/6a675e4687b94a25acb81f18acee0741 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/6a675e4687b94a25acb81f18acee0741 2024-11-23T05:41:05,941 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/6a675e4687b94a25acb81f18acee0741, entries=150, sequenceid=421, filesize=12.0 K 2024-11-23T05:41:05,942 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/.tmp/C/affcf173c2434923a5efb6e90261f69e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/affcf173c2434923a5efb6e90261f69e 2024-11-23T05:41:05,948 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/affcf173c2434923a5efb6e90261f69e, entries=150, sequenceid=421, filesize=12.0 K 2024-11-23T05:41:05,949 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6fed8128b8c4dd3776e5696d683dbd53 in 475ms, sequenceid=421, compaction requested=true 2024-11-23T05:41:05,951 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/ef6a8452e33d469bb9d759463f60477f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a8ede2a63a19443db15e397b8e863b22, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/683d62f6b14347ca9894bf454abe1cb1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/e4787585633542078cde5625f0350e0b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/366b7f5c99314e589c3ba35ebd2ae5c3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96a73dfbf01e4cdda503ab66073197b5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5d95bd70e2be4663b53557730d1690fd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/aa0056041813471090fd6f4014a6846e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/b4da424caff94cefadb7e4a7b240a97c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96be78beb8c2479b88008097ae0e9b54, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a3c4a003818d414a9799a039a94e3bc0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/62177c0df8e1496ab7148f2d6ac42e8b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c4b9e01777b34b0c8d7aa8d5aad363c3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/fdaa816e3a9945d8ad405329f20cd811, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75d9f8d1f4af4e63997fb4c786326d81, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/05cf0049e43c440ebe431545ed689633, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75a0d4a2b2be47e49ecee8de4f5bcdea, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5a64a6f1b0284c59a796117eeec61f0a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/d3e77694277747dc9ab88e3f7a17f86b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/1109d03a77d94d68b2c697d335dfdf8c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/f394a2a56a6a438da61760f28af555e9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0fc8d507b4894d07a145769036deaecd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9eec56823df041b8a4b27f4ed2c433b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/91ff9f0e389144e2b190042b3bb39499, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0bc684943b314ecdb1bedf88d8e080fe, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4f5787a5d38a4175bdfbbcb067b68a6b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c0d0ad36f17e41ec9d5bfec2f250abb1] to archive 2024-11-23T05:41:05,953 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:05,954 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/ef6a8452e33d469bb9d759463f60477f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/ef6a8452e33d469bb9d759463f60477f 2024-11-23T05:41:05,956 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a8ede2a63a19443db15e397b8e863b22 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a8ede2a63a19443db15e397b8e863b22 2024-11-23T05:41:05,957 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/683d62f6b14347ca9894bf454abe1cb1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/683d62f6b14347ca9894bf454abe1cb1 2024-11-23T05:41:05,959 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/e4787585633542078cde5625f0350e0b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/e4787585633542078cde5625f0350e0b 2024-11-23T05:41:05,960 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/366b7f5c99314e589c3ba35ebd2ae5c3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/366b7f5c99314e589c3ba35ebd2ae5c3 2024-11-23T05:41:05,961 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96a73dfbf01e4cdda503ab66073197b5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96a73dfbf01e4cdda503ab66073197b5 2024-11-23T05:41:05,963 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5d95bd70e2be4663b53557730d1690fd to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5d95bd70e2be4663b53557730d1690fd 2024-11-23T05:41:05,964 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/aa0056041813471090fd6f4014a6846e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/aa0056041813471090fd6f4014a6846e 2024-11-23T05:41:05,965 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/b4da424caff94cefadb7e4a7b240a97c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/b4da424caff94cefadb7e4a7b240a97c 2024-11-23T05:41:05,966 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96be78beb8c2479b88008097ae0e9b54 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/96be78beb8c2479b88008097ae0e9b54 2024-11-23T05:41:05,967 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a3c4a003818d414a9799a039a94e3bc0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/a3c4a003818d414a9799a039a94e3bc0 2024-11-23T05:41:05,968 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/62177c0df8e1496ab7148f2d6ac42e8b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/62177c0df8e1496ab7148f2d6ac42e8b 2024-11-23T05:41:05,969 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c4b9e01777b34b0c8d7aa8d5aad363c3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c4b9e01777b34b0c8d7aa8d5aad363c3 2024-11-23T05:41:05,970 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/fdaa816e3a9945d8ad405329f20cd811 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/fdaa816e3a9945d8ad405329f20cd811 2024-11-23T05:41:05,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75d9f8d1f4af4e63997fb4c786326d81 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75d9f8d1f4af4e63997fb4c786326d81 2024-11-23T05:41:05,972 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/05cf0049e43c440ebe431545ed689633 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/05cf0049e43c440ebe431545ed689633 2024-11-23T05:41:05,973 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75a0d4a2b2be47e49ecee8de4f5bcdea to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/75a0d4a2b2be47e49ecee8de4f5bcdea 2024-11-23T05:41:05,974 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5a64a6f1b0284c59a796117eeec61f0a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/5a64a6f1b0284c59a796117eeec61f0a 2024-11-23T05:41:05,975 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/d3e77694277747dc9ab88e3f7a17f86b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/d3e77694277747dc9ab88e3f7a17f86b 2024-11-23T05:41:05,976 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/1109d03a77d94d68b2c697d335dfdf8c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/1109d03a77d94d68b2c697d335dfdf8c 2024-11-23T05:41:05,977 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/f394a2a56a6a438da61760f28af555e9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/f394a2a56a6a438da61760f28af555e9 2024-11-23T05:41:05,978 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0fc8d507b4894d07a145769036deaecd to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0fc8d507b4894d07a145769036deaecd 2024-11-23T05:41:05,979 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9eec56823df041b8a4b27f4ed2c433b7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9eec56823df041b8a4b27f4ed2c433b7 2024-11-23T05:41:05,981 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/91ff9f0e389144e2b190042b3bb39499 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/91ff9f0e389144e2b190042b3bb39499 2024-11-23T05:41:05,982 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0bc684943b314ecdb1bedf88d8e080fe to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/0bc684943b314ecdb1bedf88d8e080fe 2024-11-23T05:41:05,984 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4f5787a5d38a4175bdfbbcb067b68a6b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4f5787a5d38a4175bdfbbcb067b68a6b 2024-11-23T05:41:05,985 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c0d0ad36f17e41ec9d5bfec2f250abb1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/c0d0ad36f17e41ec9d5bfec2f250abb1 2024-11-23T05:41:05,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/72adc6dbbdeb42c8b9dafca84cd3c167, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0e1796d13d90473598561ca5ba1d8460, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/44b3ae27e762478db34b38c88dc7c6cd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c167016c11a445fea1579ec06678084f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9d4391a463294005b7e09e62b1899e0c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/84857c2ac9ac421f9d06847f96368bb2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/46e3f14bab3f447da33ddcbb70b873bc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/d4d961c923c94ceaace68a46c179380c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7840faf403304c23b98a80e7ff8dfeaa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7151e5f859f64ec79ff8ccb8a8a4677a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/8a5c3c81cd1e4b01bbf939a3bc1e4155, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9b660f44a2fa4ccfbfb67b3a2f6f4092, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/5517a513958047e388882c61b1affcfb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/ebd9be39689e47bdaa289e26dfc180b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e581eca25bfc4913bd6300c236ad078d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/316aaf3d2bc3446bb7d743d6fe711d2c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/395156a5568047e2bae6d10cd47ede6f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a7165a6c7845495d94e1931b08640894, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a1da071df45c42949a17ec62d7f3e14a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c5f2ffc514af433294a6caa89edabb43, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3bad8b20e86b4b8e8493b009e241a034, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0343a22142764e84855668aca24e3d03, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3cde67bf853d4616b939f062f626620c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e501268c9f7e49ddb95481df0902921b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/27b7b11c1b9d4975987cd621cd70522c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c18cf0f5ef6d45f8ac45b60c78fe11d1] to archive 2024-11-23T05:41:05,987 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:05,990 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/72adc6dbbdeb42c8b9dafca84cd3c167 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/72adc6dbbdeb42c8b9dafca84cd3c167 2024-11-23T05:41:05,995 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0e1796d13d90473598561ca5ba1d8460 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0e1796d13d90473598561ca5ba1d8460 2024-11-23T05:41:05,997 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/44b3ae27e762478db34b38c88dc7c6cd to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/44b3ae27e762478db34b38c88dc7c6cd 2024-11-23T05:41:05,998 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c167016c11a445fea1579ec06678084f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c167016c11a445fea1579ec06678084f 2024-11-23T05:41:05,999 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9d4391a463294005b7e09e62b1899e0c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9d4391a463294005b7e09e62b1899e0c 2024-11-23T05:41:06,001 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/84857c2ac9ac421f9d06847f96368bb2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/84857c2ac9ac421f9d06847f96368bb2 2024-11-23T05:41:06,002 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/46e3f14bab3f447da33ddcbb70b873bc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/46e3f14bab3f447da33ddcbb70b873bc 2024-11-23T05:41:06,003 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/d4d961c923c94ceaace68a46c179380c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/d4d961c923c94ceaace68a46c179380c 2024-11-23T05:41:06,004 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7840faf403304c23b98a80e7ff8dfeaa to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7840faf403304c23b98a80e7ff8dfeaa 2024-11-23T05:41:06,005 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7151e5f859f64ec79ff8ccb8a8a4677a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/7151e5f859f64ec79ff8ccb8a8a4677a 2024-11-23T05:41:06,006 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/8a5c3c81cd1e4b01bbf939a3bc1e4155 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/8a5c3c81cd1e4b01bbf939a3bc1e4155 2024-11-23T05:41:06,007 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9b660f44a2fa4ccfbfb67b3a2f6f4092 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/9b660f44a2fa4ccfbfb67b3a2f6f4092 2024-11-23T05:41:06,009 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/5517a513958047e388882c61b1affcfb to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/5517a513958047e388882c61b1affcfb 2024-11-23T05:41:06,010 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/ebd9be39689e47bdaa289e26dfc180b7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/ebd9be39689e47bdaa289e26dfc180b7 2024-11-23T05:41:06,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e581eca25bfc4913bd6300c236ad078d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e581eca25bfc4913bd6300c236ad078d 2024-11-23T05:41:06,011 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/316aaf3d2bc3446bb7d743d6fe711d2c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/316aaf3d2bc3446bb7d743d6fe711d2c 2024-11-23T05:41:06,012 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/395156a5568047e2bae6d10cd47ede6f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/395156a5568047e2bae6d10cd47ede6f 2024-11-23T05:41:06,013 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a7165a6c7845495d94e1931b08640894 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a7165a6c7845495d94e1931b08640894 2024-11-23T05:41:06,014 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a8a73f5d4c0c4a47bcfa3e7a72f4a0c8 2024-11-23T05:41:06,015 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a1da071df45c42949a17ec62d7f3e14a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/a1da071df45c42949a17ec62d7f3e14a 2024-11-23T05:41:06,016 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c5f2ffc514af433294a6caa89edabb43 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c5f2ffc514af433294a6caa89edabb43 2024-11-23T05:41:06,017 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3bad8b20e86b4b8e8493b009e241a034 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3bad8b20e86b4b8e8493b009e241a034 2024-11-23T05:41:06,018 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0343a22142764e84855668aca24e3d03 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/0343a22142764e84855668aca24e3d03 2024-11-23T05:41:06,019 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3cde67bf853d4616b939f062f626620c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/3cde67bf853d4616b939f062f626620c 2024-11-23T05:41:06,020 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e501268c9f7e49ddb95481df0902921b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/e501268c9f7e49ddb95481df0902921b 2024-11-23T05:41:06,021 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/27b7b11c1b9d4975987cd621cd70522c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/27b7b11c1b9d4975987cd621cd70522c 2024-11-23T05:41:06,022 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c18cf0f5ef6d45f8ac45b60c78fe11d1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/c18cf0f5ef6d45f8ac45b60c78fe11d1 2024-11-23T05:41:06,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/da01845e9cde4866bb3ef2373990191d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/514e815c87e54e66bfa3e7f30d53c008, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffb5e51e3dbf4b8c9ee7664dca7800f0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/026d3da8164d47d19376f48439423dcf, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1572f400aa824b729f763e5ac71cee1c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9e89293db84a4d4f99e5428bcdda9800, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/8ecb9147cf5d415c9fe7dffeb8518d96, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1ea175576e5d4111861bdc1529d51b79, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b28f4e8e8ccd4d11879878704da11c39, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/e0b579c7c46240cdbb6856b5ac4e937a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa7a08a0a7d84c1d846a2ed0714b875f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa81dc225e3845d1bea4e505812bfdb4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/0a62db71fcb64c9eb6b00802ab655bee, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/d0c76d4ec1db48fab6197920a833b29f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3184060ed6a34116a55f7570bb642ea9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9cadfb8e0c534d4db8a51936fc39067f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b3da6d269e234959821ab56114edf2e0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3e91d672462e443d83d4c0adb38bc6f7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffba094c8f6348fc8747c3c19d157249, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/403f3bf9169247dfb7be3d432c7f8e6a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/f27d851276164ab88324b6d07ac6f00e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/5f86470834d74182b836f5e1907ed019, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/c5bae7caf77e4c00ac52293be1831200, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/89cab5dd1d494836bf9f0568eb508cef, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/7527290d06414ac9a7dbc7a82bb81446, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/10a68370460948fdb7cdb225ad66d993, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/613ee014d56440c58359713aecaa2d79] to archive 2024-11-23T05:41:06,024 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:06,026 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/da01845e9cde4866bb3ef2373990191d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/da01845e9cde4866bb3ef2373990191d 2024-11-23T05:41:06,027 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/514e815c87e54e66bfa3e7f30d53c008 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/514e815c87e54e66bfa3e7f30d53c008 2024-11-23T05:41:06,028 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffb5e51e3dbf4b8c9ee7664dca7800f0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffb5e51e3dbf4b8c9ee7664dca7800f0 2024-11-23T05:41:06,029 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/026d3da8164d47d19376f48439423dcf to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/026d3da8164d47d19376f48439423dcf 2024-11-23T05:41:06,030 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1572f400aa824b729f763e5ac71cee1c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1572f400aa824b729f763e5ac71cee1c 2024-11-23T05:41:06,031 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9e89293db84a4d4f99e5428bcdda9800 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9e89293db84a4d4f99e5428bcdda9800 2024-11-23T05:41:06,032 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/8ecb9147cf5d415c9fe7dffeb8518d96 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/8ecb9147cf5d415c9fe7dffeb8518d96 2024-11-23T05:41:06,033 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1ea175576e5d4111861bdc1529d51b79 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/1ea175576e5d4111861bdc1529d51b79 2024-11-23T05:41:06,034 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b28f4e8e8ccd4d11879878704da11c39 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b28f4e8e8ccd4d11879878704da11c39 2024-11-23T05:41:06,035 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/e0b579c7c46240cdbb6856b5ac4e937a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/e0b579c7c46240cdbb6856b5ac4e937a 2024-11-23T05:41:06,036 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa7a08a0a7d84c1d846a2ed0714b875f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa7a08a0a7d84c1d846a2ed0714b875f 2024-11-23T05:41:06,037 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa81dc225e3845d1bea4e505812bfdb4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/fa81dc225e3845d1bea4e505812bfdb4 2024-11-23T05:41:06,038 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/0a62db71fcb64c9eb6b00802ab655bee to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/0a62db71fcb64c9eb6b00802ab655bee 2024-11-23T05:41:06,039 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/d0c76d4ec1db48fab6197920a833b29f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/d0c76d4ec1db48fab6197920a833b29f 2024-11-23T05:41:06,040 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3184060ed6a34116a55f7570bb642ea9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3184060ed6a34116a55f7570bb642ea9 2024-11-23T05:41:06,042 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9cadfb8e0c534d4db8a51936fc39067f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/9cadfb8e0c534d4db8a51936fc39067f 2024-11-23T05:41:06,043 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b3da6d269e234959821ab56114edf2e0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/b3da6d269e234959821ab56114edf2e0 2024-11-23T05:41:06,044 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3e91d672462e443d83d4c0adb38bc6f7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/3e91d672462e443d83d4c0adb38bc6f7 2024-11-23T05:41:06,045 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffba094c8f6348fc8747c3c19d157249 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/ffba094c8f6348fc8747c3c19d157249 2024-11-23T05:41:06,046 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/403f3bf9169247dfb7be3d432c7f8e6a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/403f3bf9169247dfb7be3d432c7f8e6a 2024-11-23T05:41:06,047 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/f27d851276164ab88324b6d07ac6f00e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/f27d851276164ab88324b6d07ac6f00e 2024-11-23T05:41:06,048 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/5f86470834d74182b836f5e1907ed019 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/5f86470834d74182b836f5e1907ed019 2024-11-23T05:41:06,050 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/c5bae7caf77e4c00ac52293be1831200 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/c5bae7caf77e4c00ac52293be1831200 2024-11-23T05:41:06,051 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/89cab5dd1d494836bf9f0568eb508cef to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/89cab5dd1d494836bf9f0568eb508cef 2024-11-23T05:41:06,052 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/7527290d06414ac9a7dbc7a82bb81446 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/7527290d06414ac9a7dbc7a82bb81446 2024-11-23T05:41:06,053 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/10a68370460948fdb7cdb225ad66d993 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/10a68370460948fdb7cdb225ad66d993 2024-11-23T05:41:06,054 DEBUG [StoreCloser-TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/613ee014d56440c58359713aecaa2d79 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/613ee014d56440c58359713aecaa2d79 2024-11-23T05:41:06,058 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/recovered.edits/424.seqid, newMaxSeqId=424, maxSeqId=4 2024-11-23T05:41:06,059 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. 2024-11-23T05:41:06,059 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] regionserver.HRegion(1676): Region close journal for 6fed8128b8c4dd3776e5696d683dbd53: Waiting for close lock at 1732340465474Running coprocessor pre-close hooks at 1732340465474Disabling compacts and flushes for region at 1732340465474Disabling writes for close at 1732340465474Obtaining lock to block concurrent updates at 1732340465474Preparing flush snapshotting stores in 6fed8128b8c4dd3776e5696d683dbd53 at 1732340465474Finished memstore snapshotting TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732340465474Flushing stores of TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53. at 1732340465474Flushing 6fed8128b8c4dd3776e5696d683dbd53/A: creating writer at 1732340465475 (+1 ms)Flushing mob file 6fed8128b8c4dd3776e5696d683dbd53/A: appending metadata at 1732340465479 (+4 ms)Flushing mob file 6fed8128b8c4dd3776e5696d683dbd53/A: closing flushed file at 1732340465479Flushing 6fed8128b8c4dd3776e5696d683dbd53/A: appending metadata at 1732340465889 (+410 ms)Flushing 6fed8128b8c4dd3776e5696d683dbd53/A: closing flushed file at 1732340465890 (+1 ms)Flushing 6fed8128b8c4dd3776e5696d683dbd53/B: creating writer at 1732340465899 (+9 ms)Flushing 6fed8128b8c4dd3776e5696d683dbd53/B: appending metadata at 1732340465901 (+2 ms)Flushing 6fed8128b8c4dd3776e5696d683dbd53/B: closing flushed file at 1732340465901Flushing 6fed8128b8c4dd3776e5696d683dbd53/C: creating writer at 1732340465912 (+11 ms)Flushing 6fed8128b8c4dd3776e5696d683dbd53/C: appending metadata at 1732340465915 (+3 ms)Flushing 6fed8128b8c4dd3776e5696d683dbd53/C: closing flushed file at 1732340465915Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6e23b337: reopening flushed file at 1732340465929 (+14 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3f1cf82d: reopening flushed file at 1732340465935 (+6 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5c1057c7: reopening flushed file at 1732340465942 (+7 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 6fed8128b8c4dd3776e5696d683dbd53 in 475ms, sequenceid=421, compaction requested=true at 1732340465949 (+7 ms)Writing region close event to WAL at 1732340466055 (+106 ms)Running coprocessor post-close hooks at 1732340466059 (+4 ms)Closed at 1732340466059 2024-11-23T05:41:06,061 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=65}] handler.UnassignRegionHandler(157): Closed 6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,062 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=64 updating hbase:meta row=6fed8128b8c4dd3776e5696d683dbd53, regionState=CLOSED 2024-11-23T05:41:06,063 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=65, ppid=64, state=RUNNABLE, hasLock=false; CloseRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:41:06,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=65, resume processing ppid=64 2024-11-23T05:41:06,066 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=65, ppid=64, state=SUCCESS, hasLock=false; CloseRegionProcedure 6fed8128b8c4dd3776e5696d683dbd53, server=9abfc7c2569e,45719,1732340394120 in 743 msec 2024-11-23T05:41:06,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=64, resume processing ppid=63 2024-11-23T05:41:06,067 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=64, ppid=63, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=6fed8128b8c4dd3776e5696d683dbd53, UNASSIGN in 748 msec 2024-11-23T05:41:06,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=63, resume processing ppid=62 2024-11-23T05:41:06,069 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=63, ppid=62, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 754 msec 2024-11-23T05:41:06,071 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340466071"}]},"ts":"1732340466071"} 2024-11-23T05:41:06,072 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T05:41:06,072 INFO [PEWorker-2 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T05:41:06,075 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=62, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 765 msec 2024-11-23T05:41:06,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=62 2024-11-23T05:41:06,446 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:06,447 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T05:41:06,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:06,449 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=66, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:06,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T05:41:06,450 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=66, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:06,454 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,459 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/recovered.edits] 2024-11-23T05:41:06,463 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/15ce1a777a964c6ca01c0a6450a697d7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/15ce1a777a964c6ca01c0a6450a697d7 2024-11-23T05:41:06,464 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4d6918587cb044d08a06137eb4bec1b3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/4d6918587cb044d08a06137eb4bec1b3 2024-11-23T05:41:06,466 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9bc737f3439b4afc829ddc081fef0610 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/A/9bc737f3439b4afc829ddc081fef0610 2024-11-23T05:41:06,468 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/43914c6f1b9b4134bee2004afaaf1904 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/43914c6f1b9b4134bee2004afaaf1904 2024-11-23T05:41:06,469 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/6a675e4687b94a25acb81f18acee0741 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/6a675e4687b94a25acb81f18acee0741 2024-11-23T05:41:06,471 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/df96e845502f41d7be0af56edc81794d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/B/df96e845502f41d7be0af56edc81794d 2024-11-23T05:41:06,473 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/04c00c37a6ae4404bff89536363c82ad to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/04c00c37a6ae4404bff89536363c82ad 2024-11-23T05:41:06,474 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/987d9ea825854b4bb244c862ce361c74 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/987d9ea825854b4bb244c862ce361c74 2024-11-23T05:41:06,475 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/affcf173c2434923a5efb6e90261f69e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/C/affcf173c2434923a5efb6e90261f69e 2024-11-23T05:41:06,477 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/recovered.edits/424.seqid to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53/recovered.edits/424.seqid 2024-11-23T05:41:06,478 DEBUG [HFileArchiver-2 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,478 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T05:41:06,479 DEBUG [PEWorker-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T05:41:06,480 DEBUG [PEWorker-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T05:41:06,488 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112300156361806342ed8a5bae744d20eb74_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112300156361806342ed8a5bae744d20eb74_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,489 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231d67e382af97461089d1fc52318dbe8e_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411231d67e382af97461089d1fc52318dbe8e_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,491 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232357ce8acf634989aa3f8b94607dad53_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232357ce8acf634989aa3f8b94607dad53_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,493 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112323f73eb39f2e431eae6591c1f5ef3c1e_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112323f73eb39f2e431eae6591c1f5ef3c1e_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,494 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232b86c960cc264af890be84c90c228981_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232b86c960cc264af890be84c90c228981_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,496 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232f9dc8e8c3094fb497e7f29cce3dffd1_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232f9dc8e8c3094fb497e7f29cce3dffd1_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,497 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232fa4cd82cd374e6aac8b8863ec7a18ec_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411232fa4cd82cd374e6aac8b8863ec7a18ec_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,499 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235ade59e7bcf240cd8b05a559a5c67881_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411235ade59e7bcf240cd8b05a559a5c67881_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,500 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237c92612dd7014e27a85ed6e1b5669816_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237c92612dd7014e27a85ed6e1b5669816_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,501 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112380b6fa2ef0cb49f1b090f5216d59f10f_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112380b6fa2ef0cb49f1b090f5216d59f10f_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,503 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123950e712c41fd4bb99f2c0c57981f9ed8_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123950e712c41fd4bb99f2c0c57981f9ed8_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,505 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123abf7c20e768340e69957da1e0a1a8286_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123abf7c20e768340e69957da1e0a1a8286_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,506 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ad7ac8f709d941b2a99c9f7d771b79ef_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ad7ac8f709d941b2a99c9f7d771b79ef_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,507 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b433cfae874c4fa2bbba409fc1c6833d_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b433cfae874c4fa2bbba409fc1c6833d_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,508 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c51d2c628eaa4499b086afaf07dabc20_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c51d2c628eaa4499b086afaf07dabc20_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,509 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c63fd9fadc7b4065a87b0e362fc51852_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123c63fd9fadc7b4065a87b0e362fc51852_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,510 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d597e632acab4f60b2cb239444c1fc00_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d597e632acab4f60b2cb239444c1fc00_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,511 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d9114b4093a840c39e9b31388bbfe79d_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d9114b4093a840c39e9b31388bbfe79d_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,512 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dee5732d95db4c65a9b435ca18ee5a97_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dee5732d95db4c65a9b435ca18ee5a97_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,513 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e383108738ff4d6497a0cb6a177ad9f2_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e383108738ff4d6497a0cb6a177ad9f2_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eeb7f98333704d3a8dc86e6104514b99_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eeb7f98333704d3a8dc86e6104514b99_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,514 DEBUG [PEWorker-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ef550911c11a4d62bfbf4c9d2f536386_6fed8128b8c4dd3776e5696d683dbd53 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ef550911c11a4d62bfbf4c9d2f536386_6fed8128b8c4dd3776e5696d683dbd53 2024-11-23T05:41:06,515 DEBUG [PEWorker-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T05:41:06,517 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=66, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:06,519 WARN [PEWorker-4 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T05:41:06,521 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T05:41:06,522 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=66, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:06,522 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T05:41:06,523 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732340466522"}]},"ts":"9223372036854775807"} 2024-11-23T05:41:06,525 INFO [PEWorker-4 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T05:41:06,525 DEBUG [PEWorker-4 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 6fed8128b8c4dd3776e5696d683dbd53, NAME => 'TestAcidGuarantees,,1732340432171.6fed8128b8c4dd3776e5696d683dbd53.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T05:41:06,525 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T05:41:06,525 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732340466525"}]},"ts":"9223372036854775807"} 2024-11-23T05:41:06,527 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T05:41:06,528 DEBUG [PEWorker-4 {}] procedure.DeleteTableProcedure(133): Finished pid=66, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:06,529 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=66, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-23T05:41:06,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=66 2024-11-23T05:41:06,556 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T05:41:06,556 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:06,565 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobMixedAtomicity Thread=238 (was 237) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/data/data1 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935771990_22 at /127.0.0.1:45570 [Waiting for operation #15] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-5 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: Async disk worker #0 for volume /home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/data/data2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-6 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: ProcedureExecutor-Async-Task-Executor-7 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-2 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=451 (was 455), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=524 (was 497) - SystemLoadAverage LEAK? -, ProcessCount=11 (was 11), AvailableMemoryMB=8884 (was 8426) - AvailableMemoryMB LEAK? - 2024-11-23T05:41:06,572 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=238, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=524, ProcessCount=11, AvailableMemoryMB=8883 2024-11-23T05:41:06,573 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:41:06,574 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:41:06,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:06,576 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:41:06,576 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:06,576 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 67 2024-11-23T05:41:06,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T05:41:06,577 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:41:06,582 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742070_1246 (size=959) 2024-11-23T05:41:06,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T05:41:06,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T05:41:06,984 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:41:06,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742071_1247 (size=53) 2024-11-23T05:41:07,206 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T05:41:07,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:41:07,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 52e4a67ae2cd3aaf02799cd9a79db77f, disabling compactions & flushes 2024-11-23T05:41:07,390 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:07,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:07,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. after waiting 0 ms 2024-11-23T05:41:07,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:07,390 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:07,390 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 52e4a67ae2cd3aaf02799cd9a79db77f: Waiting for close lock at 1732340467390Disabling compacts and flushes for region at 1732340467390Disabling writes for close at 1732340467390Writing region close event to WAL at 1732340467390Closed at 1732340467390 2024-11-23T05:41:07,391 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:41:07,392 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732340467392"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732340467392"}]},"ts":"1732340467392"} 2024-11-23T05:41:07,394 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:41:07,395 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:41:07,395 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340467395"}]},"ts":"1732340467395"} 2024-11-23T05:41:07,397 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T05:41:07,397 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, ASSIGN}] 2024-11-23T05:41:07,398 INFO [PEWorker-3 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, ASSIGN 2024-11-23T05:41:07,399 INFO [PEWorker-3 {}] assignment.TransitRegionStateProcedure(269): Starting pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, ASSIGN; state=OFFLINE, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=false 2024-11-23T05:41:07,550 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=52e4a67ae2cd3aaf02799cd9a79db77f, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:07,551 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=68, ppid=67, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, ASSIGN because future has completed 2024-11-23T05:41:07,552 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:41:07,707 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:07,707 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7752): Opening region: {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:41:07,707 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,707 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:41:07,707 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7794): checking encryption for 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,707 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(7797): checking classloading for 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,709 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,710 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52e4a67ae2cd3aaf02799cd9a79db77f columnFamilyName A 2024-11-23T05:41:07,711 DEBUG [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:07,711 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] regionserver.HStore(327): Store=52e4a67ae2cd3aaf02799cd9a79db77f/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:07,711 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,713 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52e4a67ae2cd3aaf02799cd9a79db77f columnFamilyName B 2024-11-23T05:41:07,713 DEBUG [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:07,714 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] regionserver.HStore(327): Store=52e4a67ae2cd3aaf02799cd9a79db77f/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:07,714 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,715 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 52e4a67ae2cd3aaf02799cd9a79db77f columnFamilyName C 2024-11-23T05:41:07,715 DEBUG [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:07,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T05:41:07,716 INFO [StoreOpener-52e4a67ae2cd3aaf02799cd9a79db77f-1 {}] regionserver.HStore(327): Store=52e4a67ae2cd3aaf02799cd9a79db77f/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:07,716 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1038): replaying wal for 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,717 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,717 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,719 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1048): stopping wal replay for 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,719 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1060): Cleaning up temporary data for 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,719 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:41:07,721 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1093): writing seq id for 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,723 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:41:07,724 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1114): Opened 52e4a67ae2cd3aaf02799cd9a79db77f; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64409751, jitterRate=-0.04021991789340973}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:41:07,724 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:07,724 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegion(1006): Region open journal for 52e4a67ae2cd3aaf02799cd9a79db77f: Running coprocessor pre-open hook at 1732340467707Writing region info on filesystem at 1732340467707Initializing all the Stores at 1732340467708 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340467708Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340467709 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340467709Cleaning up temporary data from old regions at 1732340467719 (+10 ms)Running coprocessor post-open hooks at 1732340467724 (+5 ms)Region opened successfully at 1732340467724 2024-11-23T05:41:07,725 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., pid=69, masterSystemTime=1732340467703 2024-11-23T05:41:07,726 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:07,726 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=69}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:07,727 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=68 updating hbase:meta row=52e4a67ae2cd3aaf02799cd9a79db77f, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:07,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=69, ppid=68, state=RUNNABLE, hasLock=false; OpenRegionProcedure 52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:41:07,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=69, resume processing ppid=68 2024-11-23T05:41:07,730 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=69, ppid=68, state=SUCCESS, hasLock=false; OpenRegionProcedure 52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 in 177 msec 2024-11-23T05:41:07,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=68, resume processing ppid=67 2024-11-23T05:41:07,732 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=68, ppid=67, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, ASSIGN in 333 msec 2024-11-23T05:41:07,732 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:41:07,732 DEBUG [PEWorker-2 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340467732"}]},"ts":"1732340467732"} 2024-11-23T05:41:07,734 INFO [PEWorker-2 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T05:41:07,734 INFO [PEWorker-2 {}] procedure.CreateTableProcedure(89): pid=67, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:41:07,735 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=67, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1600 sec 2024-11-23T05:41:08,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=67 2024-11-23T05:41:08,727 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:08,728 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24ada2c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,728 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,728 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,730 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,730 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5aabc586, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,731 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,733 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38946, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,734 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43eec7c0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,734 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,736 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,737 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,738 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:41:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,40189,1732340393249 2024-11-23T05:41:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@4db00623 2024-11-23T05:41:08,739 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:41:08,740 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38962, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:41:08,742 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3fbd4712, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,742 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,742 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,743 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2307fb8f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,743 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,744 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,744 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,745 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@33ddc3a6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,746 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7c6f1434, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,746 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,746 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,747 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@59fd78fc, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,747 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,748 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,748 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:38998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,749 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4f070f9c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,750 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@634f1deb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,750 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,750 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,751 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2aaa7761, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,751 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,752 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,752 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39028, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,753 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ece6dc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,754 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ec2896e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,754 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,754 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,755 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@797629cd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,755 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,756 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,757 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,757 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@dc2dd40, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,758 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5155197b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,758 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,758 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,759 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@109dcb51, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,759 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,760 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,760 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39056, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,761 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e7aa3b1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,761 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2108806c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,761 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,761 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,762 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7dffcac4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,762 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,763 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39078, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,763 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5874bb21, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,764 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6935c50e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,764 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,764 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,764 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37769ecb, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,766 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39092, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,766 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1ab5792e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,767 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@767db965, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,767 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,767 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,767 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@41430e7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,768 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,769 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,769 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@753cab89, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,770 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6493a71c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,770 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,770 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,770 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6568c0c9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,771 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,772 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39142, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,772 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5fd08821, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,772 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1a1b1c7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,772 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:08,772 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:08,773 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17e2c112, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:08,773 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:08,774 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:08,774 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:39156, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:08,775 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@75ed9f33, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:08,775 DEBUG [Thread-1150 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,775 DEBUG [Thread-1163 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,775 DEBUG [Thread-1158 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,775 DEBUG [Thread-1169 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,776 DEBUG [Thread-1152 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,776 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:08,776 DEBUG [Thread-1167 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,776 DEBUG [Thread-1165 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,776 DEBUG [Thread-1156 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,776 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,777 DEBUG [Thread-1161 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,777 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34188, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,778 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:08,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34194, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34214, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,778 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34228, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,779 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=70, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:08,779 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=71, ppid=70, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:08,779 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34204, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T05:41:08,779 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,780 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34234, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,780 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,781 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34242, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,781 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,782 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,783 DEBUG [Thread-1154 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:08,783 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34240, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,783 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,784 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,784 DEBUG [Thread-1154 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:08,785 DEBUG [Thread-1154 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,785 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,786 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:08,787 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,788 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:41:08,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:08,795 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:08,795 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:41:08,817 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/732b2e2524004b4ea875af770ba4001a is 50, key is test_row_0/A:col10/1732340468794/Put/seqid=0 2024-11-23T05:41:08,822 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340528817, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,822 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,823 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340528819, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340528820, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340528823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,824 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,824 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340528823, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,825 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,827 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742072_1248 (size=12001) 2024-11-23T05:41:08,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T05:41:08,928 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340528926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,929 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340528926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,929 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,930 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,937 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T05:41:08,937 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:08,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340528937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:08,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:08,938 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:08,938 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,938 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:08,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340528937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,938 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,939 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:08,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340528937, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:08,940 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:08,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,094 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T05:41:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,094 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,094 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,095 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T05:41:09,138 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340529136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340529136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,139 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,148 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340529146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340529146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,149 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340529146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,150 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,224 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/732b2e2524004b4ea875af770ba4001a 2024-11-23T05:41:09,246 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/2c83edc37bca4acb801a85628c26af24 is 50, key is test_row_0/B:col10/1732340468794/Put/seqid=0 2024-11-23T05:41:09,248 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T05:41:09,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:09,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,248 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,248 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,249 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,264 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742073_1249 (size=12001) 2024-11-23T05:41:09,265 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/2c83edc37bca4acb801a85628c26af24 2024-11-23T05:41:09,289 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/9db772f9b8b54a0f9cd7a11d4028d13d is 50, key is test_row_0/C:col10/1732340468794/Put/seqid=0 2024-11-23T05:41:09,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742074_1250 (size=12001) 2024-11-23T05:41:09,404 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T05:41:09,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:09,404 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,404 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,405 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,406 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T05:41:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340529446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,447 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340529446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,447 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,448 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340529456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,457 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,457 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340529456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,458 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340529456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,458 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,557 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T05:41:09,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:09,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,557 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,557 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,700 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=14 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/9db772f9b8b54a0f9cd7a11d4028d13d 2024-11-23T05:41:09,710 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T05:41:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,710 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] handler.RSProcedureHandler(58): pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,710 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=71 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=71 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:09,714 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/732b2e2524004b4ea875af770ba4001a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/732b2e2524004b4ea875af770ba4001a 2024-11-23T05:41:09,718 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/732b2e2524004b4ea875af770ba4001a, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T05:41:09,719 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/2c83edc37bca4acb801a85628c26af24 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/2c83edc37bca4acb801a85628c26af24 2024-11-23T05:41:09,723 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/2c83edc37bca4acb801a85628c26af24, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T05:41:09,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/9db772f9b8b54a0f9cd7a11d4028d13d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/9db772f9b8b54a0f9cd7a11d4028d13d 2024-11-23T05:41:09,733 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/9db772f9b8b54a0f9cd7a11d4028d13d, entries=150, sequenceid=14, filesize=11.7 K 2024-11-23T05:41:09,734 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 52e4a67ae2cd3aaf02799cd9a79db77f in 939ms, sequenceid=14, compaction requested=false 2024-11-23T05:41:09,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:09,862 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=71 2024-11-23T05:41:09,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:09,863 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:41:09,867 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/c395d17abc6a4ef9ad423fcbdf18cab7 is 50, key is test_row_0/A:col10/1732340468818/Put/seqid=0 2024-11-23T05:41:09,884 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742075_1251 (size=12001) 2024-11-23T05:41:09,885 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/c395d17abc6a4ef9ad423fcbdf18cab7 2024-11-23T05:41:09,894 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/57fd4ef929b54373b9e7b1e533e394de is 50, key is test_row_0/B:col10/1732340468818/Put/seqid=0 2024-11-23T05:41:09,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742076_1252 (size=12001) 2024-11-23T05:41:09,899 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/57fd4ef929b54373b9e7b1e533e394de 2024-11-23T05:41:09,906 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/012a8c38b56240dc840840986334aa4b is 50, key is test_row_0/C:col10/1732340468818/Put/seqid=0 2024-11-23T05:41:09,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T05:41:09,922 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742077_1253 (size=12001) 2024-11-23T05:41:09,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:09,957 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340529970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,974 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,974 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340529970, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340529971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340529971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,975 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340529971, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:09,975 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:09,976 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340530086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,088 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340530086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,088 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,088 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340530086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340530086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,089 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340530086, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,089 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,297 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340530296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,298 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340530296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340530296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,299 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340530296, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,299 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,299 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,300 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340530297, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,300 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,323 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/012a8c38b56240dc840840986334aa4b 2024-11-23T05:41:10,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/c395d17abc6a4ef9ad423fcbdf18cab7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c395d17abc6a4ef9ad423fcbdf18cab7 2024-11-23T05:41:10,330 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c395d17abc6a4ef9ad423fcbdf18cab7, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T05:41:10,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/57fd4ef929b54373b9e7b1e533e394de as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/57fd4ef929b54373b9e7b1e533e394de 2024-11-23T05:41:10,334 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/57fd4ef929b54373b9e7b1e533e394de, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T05:41:10,334 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/012a8c38b56240dc840840986334aa4b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/012a8c38b56240dc840840986334aa4b 2024-11-23T05:41:10,338 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/012a8c38b56240dc840840986334aa4b, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T05:41:10,339 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 52e4a67ae2cd3aaf02799cd9a79db77f in 476ms, sequenceid=37, compaction requested=false 2024-11-23T05:41:10,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:10,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:10,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=71}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=71 2024-11-23T05:41:10,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=71 2024-11-23T05:41:10,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=71, resume processing ppid=70 2024-11-23T05:41:10,341 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=71, ppid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5610 sec 2024-11-23T05:41:10,343 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=70, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=70, table=TestAcidGuarantees in 1.5660 sec 2024-11-23T05:41:10,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:10,609 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:10,609 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T05:41:10,614 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/656a20671a7949298064b370f3b841d8 is 50, key is test_row_0/A:col10/1732340470609/Put/seqid=0 2024-11-23T05:41:10,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742078_1254 (size=12001) 2024-11-23T05:41:10,633 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340530629, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,633 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,634 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340530630, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,635 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340530631, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,636 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340530633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,637 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340530633, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,637 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,738 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340530736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,738 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,747 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340530746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,748 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340530746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,749 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,749 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340530746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340530746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,750 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=70 2024-11-23T05:41:10,926 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:10,926 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees 2024-11-23T05:41:10,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T05:41:10,928 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:10,928 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=72, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:10,928 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=73, ppid=72, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:10,948 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340530946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,948 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340530956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,957 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,957 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340530956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340530956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,958 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340530956, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:10,958 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,021 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/656a20671a7949298064b370f3b841d8 2024-11-23T05:41:11,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/aa9d0f554b5e4b44a286d9c25f2a6c1a is 50, key is test_row_0/B:col10/1732340470609/Put/seqid=0 2024-11-23T05:41:11,036 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742079_1255 (size=12001) 2024-11-23T05:41:11,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T05:41:11,080 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T05:41:11,080 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,081 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,081 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,234 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T05:41:11,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:11,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,234 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,234 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,235 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,246 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T05:41:11,259 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,259 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340531256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,260 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340531266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,269 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,269 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340531266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,269 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340531266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340531266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,270 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,387 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T05:41:11,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:11,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,387 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,387 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,437 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/aa9d0f554b5e4b44a286d9c25f2a6c1a 2024-11-23T05:41:11,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1c85a252611843ca9db152345724a354 is 50, key is test_row_0/C:col10/1732340470609/Put/seqid=0 2024-11-23T05:41:11,447 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742080_1256 (size=12001) 2024-11-23T05:41:11,539 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T05:41:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,540 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T05:41:11,586 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:41:11,692 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T05:41:11,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:11,692 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,693 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,769 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340531766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,769 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,778 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340531776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340531776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340531776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,779 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340531777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,779 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:11,780 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:11,845 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T05:41:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:11,845 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,845 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] handler.RSProcedureHandler(58): pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,846 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=73 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=73 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:11,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=54 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1c85a252611843ca9db152345724a354 2024-11-23T05:41:11,865 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/656a20671a7949298064b370f3b841d8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/656a20671a7949298064b370f3b841d8 2024-11-23T05:41:11,868 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/656a20671a7949298064b370f3b841d8, entries=150, sequenceid=54, filesize=11.7 K 2024-11-23T05:41:11,869 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/aa9d0f554b5e4b44a286d9c25f2a6c1a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9d0f554b5e4b44a286d9c25f2a6c1a 2024-11-23T05:41:11,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9d0f554b5e4b44a286d9c25f2a6c1a, entries=150, sequenceid=54, filesize=11.7 K 2024-11-23T05:41:11,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1c85a252611843ca9db152345724a354 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1c85a252611843ca9db152345724a354 2024-11-23T05:41:11,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1c85a252611843ca9db152345724a354, entries=150, sequenceid=54, filesize=11.7 K 2024-11-23T05:41:11,879 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=114.05 KB/116790 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1270ms, sequenceid=54, compaction requested=true 2024-11-23T05:41:11,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:11,879 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:11,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:11,880 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:11,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:11,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:11,880 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:11,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:11,880 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:11,881 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:11,881 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:11,881 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,881 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/732b2e2524004b4ea875af770ba4001a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c395d17abc6a4ef9ad423fcbdf18cab7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/656a20671a7949298064b370f3b841d8] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=35.2 K 2024-11-23T05:41:11,881 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:11,882 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:11,882 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,882 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/2c83edc37bca4acb801a85628c26af24, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/57fd4ef929b54373b9e7b1e533e394de, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9d0f554b5e4b44a286d9c25f2a6c1a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=35.2 K 2024-11-23T05:41:11,882 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 732b2e2524004b4ea875af770ba4001a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732340468790 2024-11-23T05:41:11,882 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2c83edc37bca4acb801a85628c26af24, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732340468790 2024-11-23T05:41:11,882 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c395d17abc6a4ef9ad423fcbdf18cab7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732340468817 2024-11-23T05:41:11,882 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 57fd4ef929b54373b9e7b1e533e394de, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732340468817 2024-11-23T05:41:11,882 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 656a20671a7949298064b370f3b841d8, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732340469969 2024-11-23T05:41:11,882 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9d0f554b5e4b44a286d9c25f2a6c1a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732340469969 2024-11-23T05:41:11,891 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#209 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:11,892 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/0c53a4605ede4e97a48bbbe07050458e is 50, key is test_row_0/B:col10/1732340470609/Put/seqid=0 2024-11-23T05:41:11,892 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#210 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:11,892 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/a6b9ade9d25440c4a38819d765a82dfc is 50, key is test_row_0/A:col10/1732340470609/Put/seqid=0 2024-11-23T05:41:11,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742081_1257 (size=12104) 2024-11-23T05:41:11,919 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742082_1258 (size=12104) 2024-11-23T05:41:11,926 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/a6b9ade9d25440c4a38819d765a82dfc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a6b9ade9d25440c4a38819d765a82dfc 2024-11-23T05:41:11,932 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into a6b9ade9d25440c4a38819d765a82dfc(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:11,932 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:11,932 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=13, startTime=1732340471879; duration=0sec 2024-11-23T05:41:11,932 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:11,932 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:11,932 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:11,933 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:11,933 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:11,934 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:11,934 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/9db772f9b8b54a0f9cd7a11d4028d13d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/012a8c38b56240dc840840986334aa4b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1c85a252611843ca9db152345724a354] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=35.2 K 2024-11-23T05:41:11,934 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9db772f9b8b54a0f9cd7a11d4028d13d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=14, earliestPutTs=1732340468790 2024-11-23T05:41:11,935 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 012a8c38b56240dc840840986334aa4b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732340468817 2024-11-23T05:41:11,935 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1c85a252611843ca9db152345724a354, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732340469969 2024-11-23T05:41:11,943 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#211 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:11,943 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/f247fc0c05494f6a81d0ebf987517c49 is 50, key is test_row_0/C:col10/1732340470609/Put/seqid=0 2024-11-23T05:41:11,978 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742083_1259 (size=12104) 2024-11-23T05:41:11,984 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/f247fc0c05494f6a81d0ebf987517c49 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f247fc0c05494f6a81d0ebf987517c49 2024-11-23T05:41:11,990 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into f247fc0c05494f6a81d0ebf987517c49(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:11,990 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:11,990 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=13, startTime=1732340471880; duration=0sec 2024-11-23T05:41:11,990 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:11,990 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:12,012 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=73 2024-11-23T05:41:12,012 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:12,012 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T05:41:12,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/21c0a712ae5b41b58ad61bbaca2927a0 is 50, key is test_row_0/A:col10/1732340470627/Put/seqid=0 2024-11-23T05:41:12,029 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742084_1260 (size=12001) 2024-11-23T05:41:12,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T05:41:12,312 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/0c53a4605ede4e97a48bbbe07050458e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/0c53a4605ede4e97a48bbbe07050458e 2024-11-23T05:41:12,319 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into 0c53a4605ede4e97a48bbbe07050458e(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:12,319 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:12,319 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=13, startTime=1732340471880; duration=0sec 2024-11-23T05:41:12,319 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:12,319 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:12,430 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/21c0a712ae5b41b58ad61bbaca2927a0 2024-11-23T05:41:12,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/edf7e48ae4ed4301821f1d45cfadda09 is 50, key is test_row_0/B:col10/1732340470627/Put/seqid=0 2024-11-23T05:41:12,452 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742085_1261 (size=12001) 2024-11-23T05:41:12,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:12,777 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:12,797 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340532794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340532794, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,798 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340532796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,799 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340532797, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,800 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,800 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340532798, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,801 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,853 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/edf7e48ae4ed4301821f1d45cfadda09 2024-11-23T05:41:12,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/ed8ecd0d09274e33a7b0505c5dd20451 is 50, key is test_row_0/C:col10/1732340470627/Put/seqid=0 2024-11-23T05:41:12,863 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742086_1262 (size=12001) 2024-11-23T05:41:12,908 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340532906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340532906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,909 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,909 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,909 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340532906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340532906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:12,910 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T05:41:13,118 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340533116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340533116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340533117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,119 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,119 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340533117, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,120 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,264 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=76 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/ed8ecd0d09274e33a7b0505c5dd20451 2024-11-23T05:41:13,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/21c0a712ae5b41b58ad61bbaca2927a0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/21c0a712ae5b41b58ad61bbaca2927a0 2024-11-23T05:41:13,272 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/21c0a712ae5b41b58ad61bbaca2927a0, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T05:41:13,272 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/edf7e48ae4ed4301821f1d45cfadda09 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/edf7e48ae4ed4301821f1d45cfadda09 2024-11-23T05:41:13,276 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/edf7e48ae4ed4301821f1d45cfadda09, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T05:41:13,277 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/ed8ecd0d09274e33a7b0505c5dd20451 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ed8ecd0d09274e33a7b0505c5dd20451 2024-11-23T05:41:13,281 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ed8ecd0d09274e33a7b0505c5dd20451, entries=150, sequenceid=76, filesize=11.7 K 2024-11-23T05:41:13,282 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1270ms, sequenceid=76, compaction requested=false 2024-11-23T05:41:13,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:13,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:13,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=73}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=73 2024-11-23T05:41:13,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=73 2024-11-23T05:41:13,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=73, resume processing ppid=72 2024-11-23T05:41:13,285 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=73, ppid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3550 sec 2024-11-23T05:41:13,286 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=72, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=72, table=TestAcidGuarantees in 2.3590 sec 2024-11-23T05:41:13,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:13,429 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:13,429 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T05:41:13,434 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/4d5b730e627d472c978161f39f690d68 is 50, key is test_row_0/A:col10/1732340472788/Put/seqid=0 2024-11-23T05:41:13,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742087_1263 (size=16681) 2024-11-23T05:41:13,476 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340533470, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,476 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,481 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340533476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,481 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340533476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,481 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340533477, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,482 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,588 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340533586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340533586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340533586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340533586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,589 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340533796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340533796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,797 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,797 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340533796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,798 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,799 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:13,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340533796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:13,799 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:13,846 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/4d5b730e627d472c978161f39f690d68 2024-11-23T05:41:13,860 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/aa9b60306a7443c49392d1ba857a10d1 is 50, key is test_row_0/B:col10/1732340472788/Put/seqid=0 2024-11-23T05:41:13,870 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742088_1264 (size=12001) 2024-11-23T05:41:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340534106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,110 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,110 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340534107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340534106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,111 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340534107, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,111 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,271 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/aa9b60306a7443c49392d1ba857a10d1 2024-11-23T05:41:14,277 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/17108ff909f545a6bee1eefcc35990ef is 50, key is test_row_0/C:col10/1732340472788/Put/seqid=0 2024-11-23T05:41:14,280 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742089_1265 (size=12001) 2024-11-23T05:41:14,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340534616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340534616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340534616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340534617, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,618 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,619 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:14,681 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=97 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/17108ff909f545a6bee1eefcc35990ef 2024-11-23T05:41:14,685 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/4d5b730e627d472c978161f39f690d68 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/4d5b730e627d472c978161f39f690d68 2024-11-23T05:41:14,689 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/4d5b730e627d472c978161f39f690d68, entries=250, sequenceid=97, filesize=16.3 K 2024-11-23T05:41:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/aa9b60306a7443c49392d1ba857a10d1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9b60306a7443c49392d1ba857a10d1 2024-11-23T05:41:14,694 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9b60306a7443c49392d1ba857a10d1, entries=150, sequenceid=97, filesize=11.7 K 2024-11-23T05:41:14,695 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/17108ff909f545a6bee1eefcc35990ef as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/17108ff909f545a6bee1eefcc35990ef 2024-11-23T05:41:14,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/17108ff909f545a6bee1eefcc35990ef, entries=150, sequenceid=97, filesize=11.7 K 2024-11-23T05:41:14,700 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=93.93 KB/96180 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1270ms, sequenceid=97, compaction requested=true 2024-11-23T05:41:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:14,700 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:14,700 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:14,700 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:14,701 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:14,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40786 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:14,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:14,701 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:14,701 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:14,701 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:14,701 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a6b9ade9d25440c4a38819d765a82dfc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/21c0a712ae5b41b58ad61bbaca2927a0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/4d5b730e627d472c978161f39f690d68] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=39.8 K 2024-11-23T05:41:14,701 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/0c53a4605ede4e97a48bbbe07050458e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/edf7e48ae4ed4301821f1d45cfadda09, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9b60306a7443c49392d1ba857a10d1] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=35.3 K 2024-11-23T05:41:14,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a6b9ade9d25440c4a38819d765a82dfc, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732340469969 2024-11-23T05:41:14,701 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0c53a4605ede4e97a48bbbe07050458e, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732340469969 2024-11-23T05:41:14,702 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting edf7e48ae4ed4301821f1d45cfadda09, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340470627 2024-11-23T05:41:14,702 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 21c0a712ae5b41b58ad61bbaca2927a0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340470627 2024-11-23T05:41:14,702 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4d5b730e627d472c978161f39f690d68, keycount=250, bloomtype=ROW, size=16.3 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732340472788 2024-11-23T05:41:14,702 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting aa9b60306a7443c49392d1ba857a10d1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732340472788 2024-11-23T05:41:14,709 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#218 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:14,709 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/11e48bcb309d4b98bf566c89934f2a25 is 50, key is test_row_0/A:col10/1732340472788/Put/seqid=0 2024-11-23T05:41:14,711 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#219 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:14,712 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/3b6d952ba1f3452e87eca5b07a795d7c is 50, key is test_row_0/B:col10/1732340472788/Put/seqid=0 2024-11-23T05:41:14,724 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742090_1266 (size=12207) 2024-11-23T05:41:14,735 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742091_1267 (size=12207) 2024-11-23T05:41:14,742 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/3b6d952ba1f3452e87eca5b07a795d7c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3b6d952ba1f3452e87eca5b07a795d7c 2024-11-23T05:41:14,747 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into 3b6d952ba1f3452e87eca5b07a795d7c(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:14,747 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:14,747 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=13, startTime=1732340474700; duration=0sec 2024-11-23T05:41:14,747 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:14,747 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:14,747 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:14,748 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:14,748 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:14,748 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:14,748 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f247fc0c05494f6a81d0ebf987517c49, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ed8ecd0d09274e33a7b0505c5dd20451, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/17108ff909f545a6bee1eefcc35990ef] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=35.3 K 2024-11-23T05:41:14,749 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting f247fc0c05494f6a81d0ebf987517c49, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=54, earliestPutTs=1732340469969 2024-11-23T05:41:14,750 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ed8ecd0d09274e33a7b0505c5dd20451, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=76, earliestPutTs=1732340470627 2024-11-23T05:41:14,750 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 17108ff909f545a6bee1eefcc35990ef, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732340472788 2024-11-23T05:41:14,757 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#220 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:14,758 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/c1858577e77145d8a821b36a4963849d is 50, key is test_row_0/C:col10/1732340472788/Put/seqid=0 2024-11-23T05:41:14,772 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742092_1268 (size=12207) 2024-11-23T05:41:14,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:14,819 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:14,819 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T05:41:14,823 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7c60b3d28aac49feb9423b2ed90fc21b is 50, key is test_row_0/A:col10/1732340473474/Put/seqid=0 2024-11-23T05:41:14,826 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742093_1269 (size=12001) 2024-11-23T05:41:14,827 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7c60b3d28aac49feb9423b2ed90fc21b 2024-11-23T05:41:14,833 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/67c1edcd1cf14fcdb861435ead37224f is 50, key is test_row_0/B:col10/1732340473474/Put/seqid=0 2024-11-23T05:41:14,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742094_1270 (size=12001) 2024-11-23T05:41:14,840 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/67c1edcd1cf14fcdb861435ead37224f 2024-11-23T05:41:14,849 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/384de24bd27d489ba48e1abcc12a0112 is 50, key is test_row_0/C:col10/1732340473474/Put/seqid=0 2024-11-23T05:41:14,866 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742095_1271 (size=12001) 2024-11-23T05:41:14,899 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:14,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340534895, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:14,900 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,007 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340535006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,008 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,096 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=72 2024-11-23T05:41:15,096 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:15,096 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:15,097 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees 2024-11-23T05:41:15,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T05:41:15,098 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:15,098 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=74, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:15,099 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=75, ppid=74, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:15,130 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/11e48bcb309d4b98bf566c89934f2a25 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/11e48bcb309d4b98bf566c89934f2a25 2024-11-23T05:41:15,134 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into 11e48bcb309d4b98bf566c89934f2a25(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:15,135 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:15,135 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=13, startTime=1732340474700; duration=0sec 2024-11-23T05:41:15,135 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:15,135 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:15,177 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/c1858577e77145d8a821b36a4963849d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/c1858577e77145d8a821b36a4963849d 2024-11-23T05:41:15,181 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into c1858577e77145d8a821b36a4963849d(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:15,181 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:15,181 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=13, startTime=1732340474700; duration=0sec 2024-11-23T05:41:15,181 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:15,181 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:15,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T05:41:15,218 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340535216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,218 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,219 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,250 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T05:41:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:15,251 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] handler.RSProcedureHandler(58): pid=75 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:15,251 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=75 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:15,252 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=75 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:15,266 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=116 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/384de24bd27d489ba48e1abcc12a0112 2024-11-23T05:41:15,271 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7c60b3d28aac49feb9423b2ed90fc21b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7c60b3d28aac49feb9423b2ed90fc21b 2024-11-23T05:41:15,274 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7c60b3d28aac49feb9423b2ed90fc21b, entries=150, sequenceid=116, filesize=11.7 K 2024-11-23T05:41:15,276 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/67c1edcd1cf14fcdb861435ead37224f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/67c1edcd1cf14fcdb861435ead37224f 2024-11-23T05:41:15,279 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/67c1edcd1cf14fcdb861435ead37224f, entries=150, sequenceid=116, filesize=11.7 K 2024-11-23T05:41:15,280 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/384de24bd27d489ba48e1abcc12a0112 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/384de24bd27d489ba48e1abcc12a0112 2024-11-23T05:41:15,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/384de24bd27d489ba48e1abcc12a0112, entries=150, sequenceid=116, filesize=11.7 K 2024-11-23T05:41:15,285 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 52e4a67ae2cd3aaf02799cd9a79db77f in 466ms, sequenceid=116, compaction requested=false 2024-11-23T05:41:15,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:15,403 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=75 2024-11-23T05:41:15,403 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:15,404 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=100.63 KB heapSize=264.42 KB 2024-11-23T05:41:15,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/004cfce606b44df8b548a34fb475e52f is 50, key is test_row_0/A:col10/1732340474888/Put/seqid=0 2024-11-23T05:41:15,411 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742096_1272 (size=12101) 2024-11-23T05:41:15,411 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/004cfce606b44df8b548a34fb475e52f 2024-11-23T05:41:15,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T05:41:15,419 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/b1e07ce221124afeb89a0acd92e95eb8 is 50, key is test_row_0/B:col10/1732340474888/Put/seqid=0 2024-11-23T05:41:15,429 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742097_1273 (size=12101) 2024-11-23T05:41:15,431 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/b1e07ce221124afeb89a0acd92e95eb8 2024-11-23T05:41:15,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/3cbd20485bae47c59724bd994a61b9f3 is 50, key is test_row_0/C:col10/1732340474888/Put/seqid=0 2024-11-23T05:41:15,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742098_1274 (size=12101) 2024-11-23T05:41:15,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:15,528 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:15,596 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340535592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,597 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,627 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340535626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340535626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,628 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2158 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:15,628 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,628 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2151 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340535636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,639 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340535636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,639 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,639 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2163 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,640 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,707 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340535706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:15,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T05:41:15,846 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=33.54 KB at sequenceid=136 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/3cbd20485bae47c59724bd994a61b9f3 2024-11-23T05:41:15,851 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/004cfce606b44df8b548a34fb475e52f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/004cfce606b44df8b548a34fb475e52f 2024-11-23T05:41:15,854 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/004cfce606b44df8b548a34fb475e52f, entries=150, sequenceid=136, filesize=11.8 K 2024-11-23T05:41:15,855 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/b1e07ce221124afeb89a0acd92e95eb8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/b1e07ce221124afeb89a0acd92e95eb8 2024-11-23T05:41:15,858 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/b1e07ce221124afeb89a0acd92e95eb8, entries=150, sequenceid=136, filesize=11.8 K 2024-11-23T05:41:15,859 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/3cbd20485bae47c59724bd994a61b9f3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3cbd20485bae47c59724bd994a61b9f3 2024-11-23T05:41:15,863 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3cbd20485bae47c59724bd994a61b9f3, entries=150, sequenceid=136, filesize=11.8 K 2024-11-23T05:41:15,863 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(3140): Finished flush of dataSize ~100.63 KB/103050, heapSize ~264.38 KB/270720, currentSize=100.63 KB/103050 for 52e4a67ae2cd3aaf02799cd9a79db77f in 459ms, sequenceid=136, compaction requested=true 2024-11-23T05:41:15,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:15,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:15,863 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=75}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=75 2024-11-23T05:41:15,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=75 2024-11-23T05:41:15,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=75, resume processing ppid=74 2024-11-23T05:41:15,866 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=75, ppid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 765 msec 2024-11-23T05:41:15,868 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=74, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=74, table=TestAcidGuarantees in 771 msec 2024-11-23T05:41:15,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:15,919 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:15,919 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=107.34 KB heapSize=282 KB 2024-11-23T05:41:15,924 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/227cf58613d2468183dab2351e78619d is 50, key is test_row_0/A:col10/1732340475580/Put/seqid=0 2024-11-23T05:41:15,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742099_1275 (size=14541) 2024-11-23T05:41:15,935 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/227cf58613d2468183dab2351e78619d 2024-11-23T05:41:15,943 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/a08b80cca9c646d5a3f4bc71d8148e71 is 50, key is test_row_0/B:col10/1732340475580/Put/seqid=0 2024-11-23T05:41:15,955 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742100_1276 (size=12151) 2024-11-23T05:41:15,955 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/a08b80cca9c646d5a3f4bc71d8148e71 2024-11-23T05:41:15,963 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/0ac3d58b4d85410abf41e58675c2aa45 is 50, key is test_row_0/C:col10/1732340475580/Put/seqid=0 2024-11-23T05:41:15,966 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742101_1277 (size=12151) 2024-11-23T05:41:15,990 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:15,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340535985, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:15,991 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:16,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:16,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340536096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:16,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=74 2024-11-23T05:41:16,236 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:16,237 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees 2024-11-23T05:41:16,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T05:41:16,238 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:16,239 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=76, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:16,239 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=77, ppid=76, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:16,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:16,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340536306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,310 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T05:41:16,367 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=35.78 KB at sequenceid=155 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/0ac3d58b4d85410abf41e58675c2aa45 2024-11-23T05:41:16,371 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/227cf58613d2468183dab2351e78619d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/227cf58613d2468183dab2351e78619d 2024-11-23T05:41:16,375 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/227cf58613d2468183dab2351e78619d, entries=200, sequenceid=155, filesize=14.2 K 2024-11-23T05:41:16,376 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/a08b80cca9c646d5a3f4bc71d8148e71 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/a08b80cca9c646d5a3f4bc71d8148e71 2024-11-23T05:41:16,379 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/a08b80cca9c646d5a3f4bc71d8148e71, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T05:41:16,380 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/0ac3d58b4d85410abf41e58675c2aa45 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/0ac3d58b4d85410abf41e58675c2aa45 2024-11-23T05:41:16,384 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/0ac3d58b4d85410abf41e58675c2aa45, entries=150, sequenceid=155, filesize=11.9 K 2024-11-23T05:41:16,385 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~107.34 KB/109920, heapSize ~281.95 KB/288720, currentSize=93.93 KB/96180 for 52e4a67ae2cd3aaf02799cd9a79db77f in 466ms, sequenceid=155, compaction requested=true 2024-11-23T05:41:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:16,385 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:16,385 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:16,385 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:16,387 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 50850 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:16,387 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:16,387 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:16,388 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/11e48bcb309d4b98bf566c89934f2a25, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7c60b3d28aac49feb9423b2ed90fc21b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/004cfce606b44df8b548a34fb475e52f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/227cf58613d2468183dab2351e78619d] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=49.7 K 2024-11-23T05:41:16,391 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:16,391 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:16,391 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:16,391 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3b6d952ba1f3452e87eca5b07a795d7c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/67c1edcd1cf14fcdb861435ead37224f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/b1e07ce221124afeb89a0acd92e95eb8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/a08b80cca9c646d5a3f4bc71d8148e71] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=47.3 K 2024-11-23T05:41:16,391 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=77 2024-11-23T05:41:16,391 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 11e48bcb309d4b98bf566c89934f2a25, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732340472788 2024-11-23T05:41:16,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:16,392 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=93.93 KB heapSize=246.84 KB 2024-11-23T05:41:16,392 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b6d952ba1f3452e87eca5b07a795d7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732340472788 2024-11-23T05:41:16,392 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c60b3d28aac49feb9423b2ed90fc21b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732340473474 2024-11-23T05:41:16,393 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 67c1edcd1cf14fcdb861435ead37224f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732340473474 2024-11-23T05:41:16,393 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 004cfce606b44df8b548a34fb475e52f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732340474859 2024-11-23T05:41:16,393 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 227cf58613d2468183dab2351e78619d, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732340475554 2024-11-23T05:41:16,394 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b1e07ce221124afeb89a0acd92e95eb8, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732340474859 2024-11-23T05:41:16,396 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a08b80cca9c646d5a3f4bc71d8148e71, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732340475580 2024-11-23T05:41:16,405 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/6fc7c49a033c4a8da27cb5d60a14f9c7 is 50, key is test_row_0/A:col10/1732340475965/Put/seqid=0 2024-11-23T05:41:16,411 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#231 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:16,412 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#232 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:16,412 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/caf33393a6f3457088e31fb928852d66 is 50, key is test_row_0/B:col10/1732340475580/Put/seqid=0 2024-11-23T05:41:16,412 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/087fb6c6435d426fb0b435796cd3f930 is 50, key is test_row_0/A:col10/1732340475580/Put/seqid=0 2024-11-23T05:41:16,416 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742102_1278 (size=12151) 2024-11-23T05:41:16,427 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742103_1279 (size=12493) 2024-11-23T05:41:16,428 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742104_1280 (size=12493) 2024-11-23T05:41:16,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T05:41:16,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:16,618 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:16,687 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:16,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340536684, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,688 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:16,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:16,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340536796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:16,799 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:16,817 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/6fc7c49a033c4a8da27cb5d60a14f9c7 2024-11-23T05:41:16,854 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/087fb6c6435d426fb0b435796cd3f930 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/087fb6c6435d426fb0b435796cd3f930 2024-11-23T05:41:16,855 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/caf33393a6f3457088e31fb928852d66 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/caf33393a6f3457088e31fb928852d66 2024-11-23T05:41:16,858 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/49108fa875334e8e951db3cb7c6c5b62 is 50, key is test_row_0/B:col10/1732340475965/Put/seqid=0 2024-11-23T05:41:16,860 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into 087fb6c6435d426fb0b435796cd3f930(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:16,860 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:16,860 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=12, startTime=1732340476385; duration=0sec 2024-11-23T05:41:16,861 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:16,861 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:16,861 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:16,862 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48460 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:16,862 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:16,862 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:16,862 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/c1858577e77145d8a821b36a4963849d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/384de24bd27d489ba48e1abcc12a0112, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3cbd20485bae47c59724bd994a61b9f3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/0ac3d58b4d85410abf41e58675c2aa45] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=47.3 K 2024-11-23T05:41:16,862 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c1858577e77145d8a821b36a4963849d, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=97, earliestPutTs=1732340472788 2024-11-23T05:41:16,862 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 384de24bd27d489ba48e1abcc12a0112, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=116, earliestPutTs=1732340473474 2024-11-23T05:41:16,863 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3cbd20485bae47c59724bd994a61b9f3, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=136, earliestPutTs=1732340474859 2024-11-23T05:41:16,863 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ac3d58b4d85410abf41e58675c2aa45, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732340475580 2024-11-23T05:41:16,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T05:41:16,868 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into caf33393a6f3457088e31fb928852d66(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:16,868 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:16,868 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=12, startTime=1732340476385; duration=0sec 2024-11-23T05:41:16,868 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:16,868 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:16,888 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742105_1281 (size=12151) 2024-11-23T05:41:16,904 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#234 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:16,905 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/6036c505fe6a48219064ed5a44e8664c is 50, key is test_row_0/C:col10/1732340475580/Put/seqid=0 2024-11-23T05:41:16,925 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742106_1282 (size=12493) 2024-11-23T05:41:16,934 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/6036c505fe6a48219064ed5a44e8664c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6036c505fe6a48219064ed5a44e8664c 2024-11-23T05:41:16,943 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into 6036c505fe6a48219064ed5a44e8664c(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:16,944 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:16,944 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=12, startTime=1732340476385; duration=0sec 2024-11-23T05:41:16,944 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:16,944 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:17,008 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340537006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,009 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:17,298 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/49108fa875334e8e951db3cb7c6c5b62 2024-11-23T05:41:17,308 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1662a7a1edfe4a38bb93c5290392df74 is 50, key is test_row_0/C:col10/1732340475965/Put/seqid=0 2024-11-23T05:41:17,316 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742107_1283 (size=12151) 2024-11-23T05:41:17,318 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340537316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,319 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:17,376 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T05:41:17,649 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340537646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,649 WARN [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4172 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,650 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:17,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340537656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340537656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,657 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340537656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,657 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4187 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,658 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:17,658 WARN [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4182 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,658 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:17,716 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=31.31 KB at sequenceid=172 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1662a7a1edfe4a38bb93c5290392df74 2024-11-23T05:41:17,722 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/6fc7c49a033c4a8da27cb5d60a14f9c7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/6fc7c49a033c4a8da27cb5d60a14f9c7 2024-11-23T05:41:17,726 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/6fc7c49a033c4a8da27cb5d60a14f9c7, entries=150, sequenceid=172, filesize=11.9 K 2024-11-23T05:41:17,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/49108fa875334e8e951db3cb7c6c5b62 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/49108fa875334e8e951db3cb7c6c5b62 2024-11-23T05:41:17,729 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/49108fa875334e8e951db3cb7c6c5b62, entries=150, sequenceid=172, filesize=11.9 K 2024-11-23T05:41:17,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1662a7a1edfe4a38bb93c5290392df74 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1662a7a1edfe4a38bb93c5290392df74 2024-11-23T05:41:17,734 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1662a7a1edfe4a38bb93c5290392df74, entries=150, sequenceid=172, filesize=11.9 K 2024-11-23T05:41:17,735 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(3140): Finished flush of dataSize ~93.93 KB/96180, heapSize ~246.80 KB/252720, currentSize=107.34 KB/109920 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1342ms, sequenceid=172, compaction requested=false 2024-11-23T05:41:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:17,735 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=77}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=77 2024-11-23T05:41:17,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=77 2024-11-23T05:41:17,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=77, resume processing ppid=76 2024-11-23T05:41:17,737 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=77, ppid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4970 sec 2024-11-23T05:41:17,739 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=76, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=76, table=TestAcidGuarantees in 1.5010 sec 2024-11-23T05:41:17,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:17,831 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:17,831 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=114.05 KB heapSize=299.58 KB 2024-11-23T05:41:17,836 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/b66121f4933a4860958100e80542ad4e is 50, key is test_row_0/A:col10/1732340477828/Put/seqid=0 2024-11-23T05:41:17,839 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742108_1284 (size=14541) 2024-11-23T05:41:17,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340537879, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,883 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:17,988 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:17,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340537986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:17,989 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:18,197 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:18,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340538196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:18,198 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:18,240 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/b66121f4933a4860958100e80542ad4e 2024-11-23T05:41:18,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/ea7e0fbe62be49f4bc127d7c31ac8cff is 50, key is test_row_0/B:col10/1732340477828/Put/seqid=0 2024-11-23T05:41:18,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742109_1285 (size=12151) 2024-11-23T05:41:18,250 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/ea7e0fbe62be49f4bc127d7c31ac8cff 2024-11-23T05:41:18,256 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a7f4aa147db9428f8e311eaf61cdfc0a is 50, key is test_row_0/C:col10/1732340477828/Put/seqid=0 2024-11-23T05:41:18,259 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742110_1286 (size=12151) 2024-11-23T05:41:18,385 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=76 2024-11-23T05:41:18,386 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:18,386 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees 2024-11-23T05:41:18,387 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T05:41:18,388 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:18,388 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=78, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:18,388 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=79, ppid=78, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:18,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T05:41:18,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:18,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 116 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340538505, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:18,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:18,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:18,507 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:18,540 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T05:41:18,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:18,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:18,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:18,541 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] handler.RSProcedureHandler(58): pid=79 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:18,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=79 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:18,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=79 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:18,659 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=38.02 KB at sequenceid=195 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a7f4aa147db9428f8e311eaf61cdfc0a 2024-11-23T05:41:18,663 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/b66121f4933a4860958100e80542ad4e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/b66121f4933a4860958100e80542ad4e 2024-11-23T05:41:18,666 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/b66121f4933a4860958100e80542ad4e, entries=200, sequenceid=195, filesize=14.2 K 2024-11-23T05:41:18,667 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/ea7e0fbe62be49f4bc127d7c31ac8cff as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/ea7e0fbe62be49f4bc127d7c31ac8cff 2024-11-23T05:41:18,670 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/ea7e0fbe62be49f4bc127d7c31ac8cff, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T05:41:18,671 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a7f4aa147db9428f8e311eaf61cdfc0a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a7f4aa147db9428f8e311eaf61cdfc0a 2024-11-23T05:41:18,678 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a7f4aa147db9428f8e311eaf61cdfc0a, entries=150, sequenceid=195, filesize=11.9 K 2024-11-23T05:41:18,679 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~114.05 KB/116790, heapSize ~299.53 KB/306720, currentSize=87.22 KB/89310 for 52e4a67ae2cd3aaf02799cd9a79db77f in 847ms, sequenceid=195, compaction requested=true 2024-11-23T05:41:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:18,679 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:18,679 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:18,679 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:18,680 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:18,680 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:18,680 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:18,680 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:18,680 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:18,680 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:18,680 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/087fb6c6435d426fb0b435796cd3f930, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/6fc7c49a033c4a8da27cb5d60a14f9c7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/b66121f4933a4860958100e80542ad4e] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=38.3 K 2024-11-23T05:41:18,680 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/caf33393a6f3457088e31fb928852d66, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/49108fa875334e8e951db3cb7c6c5b62, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/ea7e0fbe62be49f4bc127d7c31ac8cff] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=35.9 K 2024-11-23T05:41:18,681 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting caf33393a6f3457088e31fb928852d66, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732340475580 2024-11-23T05:41:18,681 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 087fb6c6435d426fb0b435796cd3f930, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732340475580 2024-11-23T05:41:18,681 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 49108fa875334e8e951db3cb7c6c5b62, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732340475965 2024-11-23T05:41:18,681 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6fc7c49a033c4a8da27cb5d60a14f9c7, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732340475965 2024-11-23T05:41:18,681 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ea7e0fbe62be49f4bc127d7c31ac8cff, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340476666 2024-11-23T05:41:18,681 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b66121f4933a4860958100e80542ad4e, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340476666 2024-11-23T05:41:18,687 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#240 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:18,688 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#239 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:18,688 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/58bcf9c139b54ce7a909bccb5fb00457 is 50, key is test_row_0/A:col10/1732340477828/Put/seqid=0 2024-11-23T05:41:18,688 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/f762bbf59f864482beea2411108a0654 is 50, key is test_row_0/B:col10/1732340477828/Put/seqid=0 2024-11-23T05:41:18,692 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742111_1287 (size=12595) 2024-11-23T05:41:18,693 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=79 2024-11-23T05:41:18,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:18,693 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T05:41:18,697 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/58bcf9c139b54ce7a909bccb5fb00457 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/58bcf9c139b54ce7a909bccb5fb00457 2024-11-23T05:41:18,701 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into 58bcf9c139b54ce7a909bccb5fb00457(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:18,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:18,701 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=13, startTime=1732340478679; duration=0sec 2024-11-23T05:41:18,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:18,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:18,701 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:18,703 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:18,703 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:18,703 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:18,703 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6036c505fe6a48219064ed5a44e8664c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1662a7a1edfe4a38bb93c5290392df74, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a7f4aa147db9428f8e311eaf61cdfc0a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=35.9 K 2024-11-23T05:41:18,703 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6036c505fe6a48219064ed5a44e8664c, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=155, earliestPutTs=1732340475580 2024-11-23T05:41:18,704 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1662a7a1edfe4a38bb93c5290392df74, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=172, earliestPutTs=1732340475965 2024-11-23T05:41:18,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742112_1288 (size=12595) 2024-11-23T05:41:18,704 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a7f4aa147db9428f8e311eaf61cdfc0a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340476666 2024-11-23T05:41:18,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T05:41:18,708 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/f762bbf59f864482beea2411108a0654 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/f762bbf59f864482beea2411108a0654 2024-11-23T05:41:18,708 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/cabada5dc6964841a27774301813cf09 is 50, key is test_row_0/A:col10/1732340477864/Put/seqid=0 2024-11-23T05:41:18,713 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into f762bbf59f864482beea2411108a0654(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:18,713 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:18,713 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=13, startTime=1732340478679; duration=0sec 2024-11-23T05:41:18,713 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:18,714 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:18,721 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#242 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:18,721 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a9e2ae12e12043a6ad9f09ef2a9833a5 is 50, key is test_row_0/C:col10/1732340477828/Put/seqid=0 2024-11-23T05:41:18,729 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742113_1289 (size=12151) 2024-11-23T05:41:18,739 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742114_1290 (size=12595) 2024-11-23T05:41:19,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T05:41:19,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:19,019 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:19,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:19,081 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 134 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340539078, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,081 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:19,129 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/cabada5dc6964841a27774301813cf09 2024-11-23T05:41:19,135 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/062384a265f2411d82bfaf17f7c130ac is 50, key is test_row_0/B:col10/1732340477864/Put/seqid=0 2024-11-23T05:41:19,138 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742115_1291 (size=12151) 2024-11-23T05:41:19,143 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a9e2ae12e12043a6ad9f09ef2a9833a5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a9e2ae12e12043a6ad9f09ef2a9833a5 2024-11-23T05:41:19,146 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into a9e2ae12e12043a6ad9f09ef2a9833a5(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:19,146 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:19,146 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=13, startTime=1732340478679; duration=0sec 2024-11-23T05:41:19,147 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:19,147 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:19,188 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:19,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 135 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340539186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,188 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:19,398 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:19,398 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 136 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340539396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,398 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:19,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T05:41:19,539 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/062384a265f2411d82bfaf17f7c130ac 2024-11-23T05:41:19,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/b6149feee28a40478864a333fcb5bf7c is 50, key is test_row_0/C:col10/1732340477864/Put/seqid=0 2024-11-23T05:41:19,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742116_1292 (size=12151) 2024-11-23T05:41:19,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:19,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 137 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340539706, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:19,707 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:19,949 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=211 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/b6149feee28a40478864a333fcb5bf7c 2024-11-23T05:41:19,953 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/cabada5dc6964841a27774301813cf09 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/cabada5dc6964841a27774301813cf09 2024-11-23T05:41:19,957 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/cabada5dc6964841a27774301813cf09, entries=150, sequenceid=211, filesize=11.9 K 2024-11-23T05:41:19,958 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/062384a265f2411d82bfaf17f7c130ac as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/062384a265f2411d82bfaf17f7c130ac 2024-11-23T05:41:19,961 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/062384a265f2411d82bfaf17f7c130ac, entries=150, sequenceid=211, filesize=11.9 K 2024-11-23T05:41:19,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/b6149feee28a40478864a333fcb5bf7c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/b6149feee28a40478864a333fcb5bf7c 2024-11-23T05:41:19,966 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/b6149feee28a40478864a333fcb5bf7c, entries=150, sequenceid=211, filesize=11.9 K 2024-11-23T05:41:19,967 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1274ms, sequenceid=211, compaction requested=false 2024-11-23T05:41:19,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:19,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:19,967 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=79}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=79 2024-11-23T05:41:19,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=79 2024-11-23T05:41:19,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=79, resume processing ppid=78 2024-11-23T05:41:19,969 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=79, ppid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5800 sec 2024-11-23T05:41:19,971 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=78, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=78, table=TestAcidGuarantees in 1.5840 sec 2024-11-23T05:41:20,218 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:20,219 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:20,219 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T05:41:20,223 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7f57cef388a24b36a0944b4f75cbd2e8 is 50, key is test_row_0/A:col10/1732340480217/Put/seqid=0 2024-11-23T05:41:20,244 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742117_1293 (size=14541) 2024-11-23T05:41:20,244 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7f57cef388a24b36a0944b4f75cbd2e8 2024-11-23T05:41:20,251 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/65393c19ad294ab383a9bd0bc607b8de is 50, key is test_row_0/B:col10/1732340480217/Put/seqid=0 2024-11-23T05:41:20,254 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742118_1294 (size=12151) 2024-11-23T05:41:20,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/65393c19ad294ab383a9bd0bc607b8de 2024-11-23T05:41:20,265 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:20,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 151 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340540262, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:20,265 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,266 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:20,269 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a0a818fc083c4434bf8be526f7831747 is 50, key is test_row_0/C:col10/1732340480217/Put/seqid=0 2024-11-23T05:41:20,273 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742119_1295 (size=12151) 2024-11-23T05:41:20,273 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=235 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a0a818fc083c4434bf8be526f7831747 2024-11-23T05:41:20,277 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7f57cef388a24b36a0944b4f75cbd2e8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7f57cef388a24b36a0944b4f75cbd2e8 2024-11-23T05:41:20,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7f57cef388a24b36a0944b4f75cbd2e8, entries=200, sequenceid=235, filesize=14.2 K 2024-11-23T05:41:20,281 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/65393c19ad294ab383a9bd0bc607b8de as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/65393c19ad294ab383a9bd0bc607b8de 2024-11-23T05:41:20,284 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/65393c19ad294ab383a9bd0bc607b8de, entries=150, sequenceid=235, filesize=11.9 K 2024-11-23T05:41:20,285 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/a0a818fc083c4434bf8be526f7831747 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a0a818fc083c4434bf8be526f7831747 2024-11-23T05:41:20,288 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a0a818fc083c4434bf8be526f7831747, entries=150, sequenceid=235, filesize=11.9 K 2024-11-23T05:41:20,289 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 52e4a67ae2cd3aaf02799cd9a79db77f in 70ms, sequenceid=235, compaction requested=true 2024-11-23T05:41:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:20,289 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:20,289 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:20,289 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:20,290 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:20,290 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39287 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:20,290 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:20,290 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:20,290 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,290 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,290 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/f762bbf59f864482beea2411108a0654, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/062384a265f2411d82bfaf17f7c130ac, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/65393c19ad294ab383a9bd0bc607b8de] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=36.0 K 2024-11-23T05:41:20,290 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/58bcf9c139b54ce7a909bccb5fb00457, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/cabada5dc6964841a27774301813cf09, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7f57cef388a24b36a0944b4f75cbd2e8] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=38.4 K 2024-11-23T05:41:20,290 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting f762bbf59f864482beea2411108a0654, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340476666 2024-11-23T05:41:20,290 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 58bcf9c139b54ce7a909bccb5fb00457, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340476666 2024-11-23T05:41:20,291 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting cabada5dc6964841a27774301813cf09, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732340477864 2024-11-23T05:41:20,291 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 062384a265f2411d82bfaf17f7c130ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732340477864 2024-11-23T05:41:20,291 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7f57cef388a24b36a0944b4f75cbd2e8, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340479061 2024-11-23T05:41:20,291 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 65393c19ad294ab383a9bd0bc607b8de, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340479061 2024-11-23T05:41:20,297 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#248 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:20,298 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#249 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:20,298 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/57f3b1b2015c405d96bd0fcfb3f78d78 is 50, key is test_row_0/A:col10/1732340480217/Put/seqid=0 2024-11-23T05:41:20,298 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/37a3d816c31b4a2582ec179611dd0df5 is 50, key is test_row_0/B:col10/1732340480217/Put/seqid=0 2024-11-23T05:41:20,301 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742120_1296 (size=12697) 2024-11-23T05:41:20,302 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742121_1297 (size=12697) 2024-11-23T05:41:20,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:20,379 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:20,380 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=87.22 KB heapSize=229.27 KB 2024-11-23T05:41:20,383 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/856f694731df4635be3128b50a5d1ab5 is 50, key is test_row_0/A:col10/1732340480378/Put/seqid=0 2024-11-23T05:41:20,386 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742122_1298 (size=12151) 2024-11-23T05:41:20,467 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:20,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 170 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340540463, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:20,467 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,468 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:20,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=78 2024-11-23T05:41:20,536 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:20,537 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees 2024-11-23T05:41:20,538 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:20,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T05:41:20,539 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=80, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:20,539 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=81, ppid=80, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:20,579 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:20,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 171 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340540576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,579 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:20,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T05:41:20,691 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:20,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:20,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,691 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,691 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,705 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/57f3b1b2015c405d96bd0fcfb3f78d78 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/57f3b1b2015c405d96bd0fcfb3f78d78 2024-11-23T05:41:20,706 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/37a3d816c31b4a2582ec179611dd0df5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/37a3d816c31b4a2582ec179611dd0df5 2024-11-23T05:41:20,710 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into 37a3d816c31b4a2582ec179611dd0df5(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:20,710 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into 57f3b1b2015c405d96bd0fcfb3f78d78(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:20,710 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:20,710 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:20,710 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=13, startTime=1732340480289; duration=0sec 2024-11-23T05:41:20,710 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=13, startTime=1732340480289; duration=0sec 2024-11-23T05:41:20,710 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:20,710 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:20,710 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:20,710 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:20,710 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:20,711 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:20,711 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:20,711 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,711 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a9e2ae12e12043a6ad9f09ef2a9833a5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/b6149feee28a40478864a333fcb5bf7c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a0a818fc083c4434bf8be526f7831747] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=36.0 K 2024-11-23T05:41:20,712 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a9e2ae12e12043a6ad9f09ef2a9833a5, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=195, earliestPutTs=1732340476666 2024-11-23T05:41:20,712 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b6149feee28a40478864a333fcb5bf7c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=211, earliestPutTs=1732340477864 2024-11-23T05:41:20,712 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a0a818fc083c4434bf8be526f7831747, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340479061 2024-11-23T05:41:20,718 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#251 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:20,718 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/fd5e723876da42f6ab0abb26d3400c72 is 50, key is test_row_0/C:col10/1732340480217/Put/seqid=0 2024-11-23T05:41:20,723 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742123_1299 (size=12697) 2024-11-23T05:41:20,787 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/856f694731df4635be3128b50a5d1ab5 2024-11-23T05:41:20,789 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:20,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 172 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340540786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:20,789 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:20,794 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/358ef23715ae4618a08463720706e0ae is 50, key is test_row_0/B:col10/1732340480378/Put/seqid=0 2024-11-23T05:41:20,797 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742124_1300 (size=12151) 2024-11-23T05:41:20,843 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,844 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T05:41:20,996 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:20,996 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:20,997 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,997 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:20,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,098 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,098 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 173 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340541095, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,098 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,127 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/fd5e723876da42f6ab0abb26d3400c72 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/fd5e723876da42f6ab0abb26d3400c72 2024-11-23T05:41:21,132 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into fd5e723876da42f6ab0abb26d3400c72(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:21,132 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:21,132 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=13, startTime=1732340480289; duration=0sec 2024-11-23T05:41:21,132 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:21,132 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:21,149 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,149 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,149 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,150 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T05:41:21,197 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/358ef23715ae4618a08463720706e0ae 2024-11-23T05:41:21,203 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/2ca4b9331d784e94967929411ff5ef59 is 50, key is test_row_0/C:col10/1732340480378/Put/seqid=0 2024-11-23T05:41:21,207 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742125_1301 (size=12151) 2024-11-23T05:41:21,302 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,302 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,302 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,303 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,454 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:21,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,455 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,455 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,607 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,607 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,607 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,608 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=29.07 KB at sequenceid=251 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/2ca4b9331d784e94967929411ff5ef59 2024-11-23T05:41:21,608 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 174 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340541606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,609 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,612 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/856f694731df4635be3128b50a5d1ab5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/856f694731df4635be3128b50a5d1ab5 2024-11-23T05:41:21,626 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/856f694731df4635be3128b50a5d1ab5, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T05:41:21,627 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/358ef23715ae4618a08463720706e0ae as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/358ef23715ae4618a08463720706e0ae 2024-11-23T05:41:21,631 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/358ef23715ae4618a08463720706e0ae, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T05:41:21,632 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/2ca4b9331d784e94967929411ff5ef59 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/2ca4b9331d784e94967929411ff5ef59 2024-11-23T05:41:21,636 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/2ca4b9331d784e94967929411ff5ef59, entries=150, sequenceid=251, filesize=11.9 K 2024-11-23T05:41:21,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~87.22 KB/89310, heapSize ~229.22 KB/234720, currentSize=114.05 KB/116790 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1258ms, sequenceid=251, compaction requested=false 2024-11-23T05:41:21,637 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:21,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:21,669 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:21,669 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=120.76 KB heapSize=317.16 KB 2024-11-23T05:41:21,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/ba13f8e57a694167be47bed923ade4b2 is 50, key is test_row_0/A:col10/1732340480447/Put/seqid=0 2024-11-23T05:41:21,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T05:41:21,678 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742126_1302 (size=12301) 2024-11-23T05:41:21,680 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/ba13f8e57a694167be47bed923ade4b2 2024-11-23T05:41:21,691 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/69f0967d45484197b020d6f8f71087a4 is 50, key is test_row_0/B:col10/1732340480447/Put/seqid=0 2024-11-23T05:41:21,695 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742127_1303 (size=12301) 2024-11-23T05:41:21,696 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340541693, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,697 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340541695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,699 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34228 deadline: 1732340541696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,700 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340541696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,700 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8224 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:21,700 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,701 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,759 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:21,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:21,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,760 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,760 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340541806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,809 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,809 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340541806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,810 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340541806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:21,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:21,912 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:21,912 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:21,912 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,913 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:21,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:22,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340542016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340542016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,019 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340542016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,019 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,065 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:22,065 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] handler.RSProcedureHandler(58): pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:22,065 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=81 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:22,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=81 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:22,096 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/69f0967d45484197b020d6f8f71087a4 2024-11-23T05:41:22,104 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/e58c319d37ed4d1d86166be53f9810e7 is 50, key is test_row_0/C:col10/1732340480447/Put/seqid=0 2024-11-23T05:41:22,107 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742128_1304 (size=12301) 2024-11-23T05:41:22,108 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=40.25 KB at sequenceid=275 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/e58c319d37ed4d1d86166be53f9810e7 2024-11-23T05:41:22,111 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/ba13f8e57a694167be47bed923ade4b2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ba13f8e57a694167be47bed923ade4b2 2024-11-23T05:41:22,115 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ba13f8e57a694167be47bed923ade4b2, entries=150, sequenceid=275, filesize=12.0 K 2024-11-23T05:41:22,116 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/69f0967d45484197b020d6f8f71087a4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/69f0967d45484197b020d6f8f71087a4 2024-11-23T05:41:22,120 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/69f0967d45484197b020d6f8f71087a4, entries=150, sequenceid=275, filesize=12.0 K 2024-11-23T05:41:22,120 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/e58c319d37ed4d1d86166be53f9810e7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/e58c319d37ed4d1d86166be53f9810e7 2024-11-23T05:41:22,123 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/e58c319d37ed4d1d86166be53f9810e7, entries=150, sequenceid=275, filesize=12.0 K 2024-11-23T05:41:22,124 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~120.76 KB/123660, heapSize ~317.11 KB/324720, currentSize=80.51 KB/82440 for 52e4a67ae2cd3aaf02799cd9a79db77f in 455ms, sequenceid=275, compaction requested=true 2024-11-23T05:41:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:22,124 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:22,124 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:22,124 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:22,125 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:22,125 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:22,125 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/37a3d816c31b4a2582ec179611dd0df5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/358ef23715ae4618a08463720706e0ae, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/69f0967d45484197b020d6f8f71087a4] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=36.3 K 2024-11-23T05:41:22,125 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/57f3b1b2015c405d96bd0fcfb3f78d78, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/856f694731df4635be3128b50a5d1ab5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ba13f8e57a694167be47bed923ade4b2] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=36.3 K 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 37a3d816c31b4a2582ec179611dd0df5, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340479061 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 57f3b1b2015c405d96bd0fcfb3f78d78, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340479061 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 856f694731df4635be3128b50a5d1ab5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732340480230 2024-11-23T05:41:22,125 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 358ef23715ae4618a08463720706e0ae, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732340480230 2024-11-23T05:41:22,126 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ba13f8e57a694167be47bed923ade4b2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732340480447 2024-11-23T05:41:22,126 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 69f0967d45484197b020d6f8f71087a4, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732340480447 2024-11-23T05:41:22,132 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#257 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:22,132 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#258 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:22,132 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/3d7f9b7a815f437b92bc3da0168d4125 is 50, key is test_row_0/B:col10/1732340480447/Put/seqid=0 2024-11-23T05:41:22,132 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/17a285802112416f909d88b145bd55c9 is 50, key is test_row_0/A:col10/1732340480447/Put/seqid=0 2024-11-23T05:41:22,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742129_1305 (size=12949) 2024-11-23T05:41:22,139 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742130_1306 (size=12949) 2024-11-23T05:41:22,217 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=81 2024-11-23T05:41:22,218 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:22,218 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=80.51 KB heapSize=211.69 KB 2024-11-23T05:41:22,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/a71d5f87376a4500ab6357c51baf3d1d is 50, key is test_row_0/A:col10/1732340481684/Put/seqid=0 2024-11-23T05:41:22,225 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742131_1307 (size=12301) 2024-11-23T05:41:22,328 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:22,328 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:22,346 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T05:41:22,356 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340542351, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,356 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,358 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340542354, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,358 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,359 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340542355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,359 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340542466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,466 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340542466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,466 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,467 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,469 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340542466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,469 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,540 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/3d7f9b7a815f437b92bc3da0168d4125 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3d7f9b7a815f437b92bc3da0168d4125 2024-11-23T05:41:22,544 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/17a285802112416f909d88b145bd55c9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/17a285802112416f909d88b145bd55c9 2024-11-23T05:41:22,544 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into 3d7f9b7a815f437b92bc3da0168d4125(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:22,545 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:22,545 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=13, startTime=1732340482124; duration=0sec 2024-11-23T05:41:22,545 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:22,545 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:22,545 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:22,546 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:22,546 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:22,546 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:22,546 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/fd5e723876da42f6ab0abb26d3400c72, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/2ca4b9331d784e94967929411ff5ef59, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/e58c319d37ed4d1d86166be53f9810e7] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=36.3 K 2024-11-23T05:41:22,546 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting fd5e723876da42f6ab0abb26d3400c72, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=235, earliestPutTs=1732340479061 2024-11-23T05:41:22,547 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ca4b9331d784e94967929411ff5ef59, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=251, earliestPutTs=1732340480230 2024-11-23T05:41:22,547 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e58c319d37ed4d1d86166be53f9810e7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732340480447 2024-11-23T05:41:22,549 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into 17a285802112416f909d88b145bd55c9(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:22,549 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:22,549 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=13, startTime=1732340482124; duration=0sec 2024-11-23T05:41:22,549 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:22,549 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:22,554 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#260 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:22,555 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/4ca513deffcc427fa7901fc9cc89fc92 is 50, key is test_row_0/C:col10/1732340480447/Put/seqid=0 2024-11-23T05:41:22,558 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742132_1308 (size=12949) 2024-11-23T05:41:22,564 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/4ca513deffcc427fa7901fc9cc89fc92 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/4ca513deffcc427fa7901fc9cc89fc92 2024-11-23T05:41:22,569 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into 4ca513deffcc427fa7901fc9cc89fc92(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:22,569 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:22,569 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=13, startTime=1732340482124; duration=0sec 2024-11-23T05:41:22,569 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:22,569 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:22,626 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/a71d5f87376a4500ab6357c51baf3d1d 2024-11-23T05:41:22,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 175 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340542626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,628 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 52e4a67ae2cd3aaf02799cd9a79db77f of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2164 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,628 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/590e53ce773c41f6b69d02af4b57e2d2 is 50, key is test_row_0/B:col10/1732340481684/Put/seqid=0 2024-11-23T05:41:22,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742133_1309 (size=12301) 2024-11-23T05:41:22,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340542676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340542676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340542676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T05:41:22,988 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340542986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340542986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,989 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340542987, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:22,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,037 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/590e53ce773c41f6b69d02af4b57e2d2 2024-11-23T05:41:23,042 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/5aad5074fdb7477d8171bbbf8143d426 is 50, key is test_row_0/C:col10/1732340481684/Put/seqid=0 2024-11-23T05:41:23,046 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742134_1310 (size=12301) 2024-11-23T05:41:23,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=290 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/5aad5074fdb7477d8171bbbf8143d426 2024-11-23T05:41:23,450 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/a71d5f87376a4500ab6357c51baf3d1d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a71d5f87376a4500ab6357c51baf3d1d 2024-11-23T05:41:23,453 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a71d5f87376a4500ab6357c51baf3d1d, entries=150, sequenceid=290, filesize=12.0 K 2024-11-23T05:41:23,454 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/590e53ce773c41f6b69d02af4b57e2d2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/590e53ce773c41f6b69d02af4b57e2d2 2024-11-23T05:41:23,457 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/590e53ce773c41f6b69d02af4b57e2d2, entries=150, sequenceid=290, filesize=12.0 K 2024-11-23T05:41:23,457 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/5aad5074fdb7477d8171bbbf8143d426 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/5aad5074fdb7477d8171bbbf8143d426 2024-11-23T05:41:23,460 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/5aad5074fdb7477d8171bbbf8143d426, entries=150, sequenceid=290, filesize=12.0 K 2024-11-23T05:41:23,461 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1243ms, sequenceid=290, compaction requested=false 2024-11-23T05:41:23,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:23,461 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:23,462 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=81}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=81 2024-11-23T05:41:23,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=81 2024-11-23T05:41:23,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=81, resume processing ppid=80 2024-11-23T05:41:23,464 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=81, ppid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.9240 sec 2024-11-23T05:41:23,466 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=80, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=80, table=TestAcidGuarantees in 2.9280 sec 2024-11-23T05:41:23,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:23,498 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:23,498 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:41:23,503 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7b09f296a26f420ea8f6d6fdb1d1e749 is 50, key is test_row_0/A:col10/1732340483497/Put/seqid=0 2024-11-23T05:41:23,506 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742135_1311 (size=14741) 2024-11-23T05:41:23,507 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7b09f296a26f420ea8f6d6fdb1d1e749 2024-11-23T05:41:23,514 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340543512, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,515 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/eb25db8f381741688d38c5c223dfecbd is 50, key is test_row_0/B:col10/1732340483497/Put/seqid=0 2024-11-23T05:41:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,515 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340543513, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,517 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,517 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340543514, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742136_1312 (size=12301) 2024-11-23T05:41:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,522 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340543626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,628 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,628 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340543626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,629 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340543626, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,629 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340543836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340543836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,837 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,837 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340543836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:23,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:23,838 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:23,920 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/eb25db8f381741688d38c5c223dfecbd 2024-11-23T05:41:23,926 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/26b9d67a84754bc1b1535d5ad157f71e is 50, key is test_row_0/C:col10/1732340483497/Put/seqid=0 2024-11-23T05:41:23,937 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742137_1313 (size=12301) 2024-11-23T05:41:24,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340544146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,148 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,148 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340544146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,149 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340544146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,149 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,330 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=317 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/26b9d67a84754bc1b1535d5ad157f71e 2024-11-23T05:41:24,333 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/7b09f296a26f420ea8f6d6fdb1d1e749 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7b09f296a26f420ea8f6d6fdb1d1e749 2024-11-23T05:41:24,336 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7b09f296a26f420ea8f6d6fdb1d1e749, entries=200, sequenceid=317, filesize=14.4 K 2024-11-23T05:41:24,337 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/eb25db8f381741688d38c5c223dfecbd as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/eb25db8f381741688d38c5c223dfecbd 2024-11-23T05:41:24,340 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/eb25db8f381741688d38c5c223dfecbd, entries=150, sequenceid=317, filesize=12.0 K 2024-11-23T05:41:24,341 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/26b9d67a84754bc1b1535d5ad157f71e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/26b9d67a84754bc1b1535d5ad157f71e 2024-11-23T05:41:24,345 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/26b9d67a84754bc1b1535d5ad157f71e, entries=150, sequenceid=317, filesize=12.0 K 2024-11-23T05:41:24,346 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 52e4a67ae2cd3aaf02799cd9a79db77f in 848ms, sequenceid=317, compaction requested=true 2024-11-23T05:41:24,346 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:24,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:24,346 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:24,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:24,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:24,346 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:24,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:24,346 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:24,347 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:24,347 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:24,347 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:24,347 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:24,347 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:24,347 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:24,347 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/17a285802112416f909d88b145bd55c9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a71d5f87376a4500ab6357c51baf3d1d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7b09f296a26f420ea8f6d6fdb1d1e749] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=39.1 K 2024-11-23T05:41:24,347 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:24,347 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3d7f9b7a815f437b92bc3da0168d4125, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/590e53ce773c41f6b69d02af4b57e2d2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/eb25db8f381741688d38c5c223dfecbd] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=36.7 K 2024-11-23T05:41:24,347 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17a285802112416f909d88b145bd55c9, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732340480447 2024-11-23T05:41:24,347 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3d7f9b7a815f437b92bc3da0168d4125, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732340480447 2024-11-23T05:41:24,348 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a71d5f87376a4500ab6357c51baf3d1d, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732340481684 2024-11-23T05:41:24,348 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 590e53ce773c41f6b69d02af4b57e2d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732340481684 2024-11-23T05:41:24,348 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b09f296a26f420ea8f6d6fdb1d1e749, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340482348 2024-11-23T05:41:24,348 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting eb25db8f381741688d38c5c223dfecbd, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340482348 2024-11-23T05:41:24,355 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#267 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:24,355 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#266 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:24,355 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/e8dd21d325984033ba1c4e72831013a2 is 50, key is test_row_0/B:col10/1732340483497/Put/seqid=0 2024-11-23T05:41:24,355 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/c2a6bf0c06ed443494b07a417bfa07ee is 50, key is test_row_0/A:col10/1732340483497/Put/seqid=0 2024-11-23T05:41:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742139_1315 (size=13051) 2024-11-23T05:41:24,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742138_1314 (size=13051) 2024-11-23T05:41:24,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:24,639 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:24,639 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:41:24,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/ab82db1fa4914dc381cf6cd2841cc99e is 50, key is test_row_0/A:col10/1732340483511/Put/seqid=0 2024-11-23T05:41:24,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742140_1316 (size=14741) 2024-11-23T05:41:24,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340544700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,705 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,705 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340544700, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,706 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=80 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340544701, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,706 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,706 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,706 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:24,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees 2024-11-23T05:41:24,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T05:41:24,708 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:24,709 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=82, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:24,709 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=83, ppid=82, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:24,711 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,711 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 185 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340544705, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,712 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,784 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/e8dd21d325984033ba1c4e72831013a2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e8dd21d325984033ba1c4e72831013a2 2024-11-23T05:41:24,784 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/c2a6bf0c06ed443494b07a417bfa07ee as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c2a6bf0c06ed443494b07a417bfa07ee 2024-11-23T05:41:24,788 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into c2a6bf0c06ed443494b07a417bfa07ee(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:24,788 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into e8dd21d325984033ba1c4e72831013a2(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:24,788 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:24,788 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:24,788 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=13, startTime=1732340484346; duration=0sec 2024-11-23T05:41:24,788 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=13, startTime=1732340484346; duration=0sec 2024-11-23T05:41:24,788 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:24,788 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:24,788 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:24,788 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:24,788 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:24,789 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:24,789 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:24,789 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:24,789 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/4ca513deffcc427fa7901fc9cc89fc92, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/5aad5074fdb7477d8171bbbf8143d426, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/26b9d67a84754bc1b1535d5ad157f71e] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=36.7 K 2024-11-23T05:41:24,789 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4ca513deffcc427fa7901fc9cc89fc92, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=275, earliestPutTs=1732340480447 2024-11-23T05:41:24,790 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5aad5074fdb7477d8171bbbf8143d426, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=290, earliestPutTs=1732340481684 2024-11-23T05:41:24,790 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 26b9d67a84754bc1b1535d5ad157f71e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340482348 2024-11-23T05:41:24,795 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#269 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:24,795 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/6b3b2834ca54492c98518711fcc53f42 is 50, key is test_row_0/C:col10/1732340483497/Put/seqid=0 2024-11-23T05:41:24,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742141_1317 (size=13051) 2024-11-23T05:41:24,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T05:41:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340544816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340544816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,818 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340544816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,818 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 186 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340544816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:24,819 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:24,861 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:24,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:24,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:24,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:24,861 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:24,861 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:24,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,014 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,014 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,014 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T05:41:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340545026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340545026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,029 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 187 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340545026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,030 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340545026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,030 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,046 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/ab82db1fa4914dc381cf6cd2841cc99e 2024-11-23T05:41:25,053 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/9bf20008675a48adba519bb86c866543 is 50, key is test_row_0/B:col10/1732340483511/Put/seqid=0 2024-11-23T05:41:25,055 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742142_1318 (size=12301) 2024-11-23T05:41:25,167 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,167 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,167 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,202 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/6b3b2834ca54492c98518711fcc53f42 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6b3b2834ca54492c98518711fcc53f42 2024-11-23T05:41:25,206 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into 6b3b2834ca54492c98518711fcc53f42(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:25,206 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:25,207 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=13, startTime=1732340484346; duration=0sec 2024-11-23T05:41:25,207 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:25,207 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:25,319 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:25,319 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,320 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,320 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,320 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T05:41:25,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340545336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,336 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340545336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,337 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 188 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340545336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,338 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,338 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340545336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,339 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,456 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/9bf20008675a48adba519bb86c866543 2024-11-23T05:41:25,463 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/070e17870ed64aa6a286c8a45a56f755 is 50, key is test_row_0/C:col10/1732340483511/Put/seqid=0 2024-11-23T05:41:25,471 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742143_1319 (size=12301) 2024-11-23T05:41:25,472 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:25,472 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,473 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,473 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,473 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,625 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,625 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,625 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,778 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,778 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] handler.RSProcedureHandler(58): pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,778 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=83 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=83 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:25,846 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T05:41:25,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340545846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 189 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340545846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,848 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340545846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,848 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,849 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340545846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:25,849 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:25,871 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=330 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/070e17870ed64aa6a286c8a45a56f755 2024-11-23T05:41:25,875 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/ab82db1fa4914dc381cf6cd2841cc99e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ab82db1fa4914dc381cf6cd2841cc99e 2024-11-23T05:41:25,878 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ab82db1fa4914dc381cf6cd2841cc99e, entries=200, sequenceid=330, filesize=14.4 K 2024-11-23T05:41:25,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/9bf20008675a48adba519bb86c866543 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/9bf20008675a48adba519bb86c866543 2024-11-23T05:41:25,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/9bf20008675a48adba519bb86c866543, entries=150, sequenceid=330, filesize=12.0 K 2024-11-23T05:41:25,883 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/070e17870ed64aa6a286c8a45a56f755 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/070e17870ed64aa6a286c8a45a56f755 2024-11-23T05:41:25,886 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/070e17870ed64aa6a286c8a45a56f755, entries=150, sequenceid=330, filesize=12.0 K 2024-11-23T05:41:25,887 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1248ms, sequenceid=330, compaction requested=false 2024-11-23T05:41:25,887 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:25,930 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=83 2024-11-23T05:41:25,931 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:25,931 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:41:25,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/2d6845fadfb34a8287802fdc3cd96df6 is 50, key is test_row_0/A:col10/1732340484704/Put/seqid=0 2024-11-23T05:41:25,948 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742144_1320 (size=12301) 2024-11-23T05:41:25,949 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/2d6845fadfb34a8287802fdc3cd96df6 2024-11-23T05:41:25,957 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/e494547dcb8b49a2804755b1e04fa631 is 50, key is test_row_0/B:col10/1732340484704/Put/seqid=0 2024-11-23T05:41:25,962 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742145_1321 (size=12301) 2024-11-23T05:41:26,362 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/e494547dcb8b49a2804755b1e04fa631 2024-11-23T05:41:26,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/f393ea911f6e4461b4605e2f21a585c6 is 50, key is test_row_0/C:col10/1732340484704/Put/seqid=0 2024-11-23T05:41:26,372 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742146_1322 (size=12301) 2024-11-23T05:41:26,773 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=356 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/f393ea911f6e4461b4605e2f21a585c6 2024-11-23T05:41:26,777 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/2d6845fadfb34a8287802fdc3cd96df6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/2d6845fadfb34a8287802fdc3cd96df6 2024-11-23T05:41:26,780 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/2d6845fadfb34a8287802fdc3cd96df6, entries=150, sequenceid=356, filesize=12.0 K 2024-11-23T05:41:26,781 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/e494547dcb8b49a2804755b1e04fa631 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e494547dcb8b49a2804755b1e04fa631 2024-11-23T05:41:26,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e494547dcb8b49a2804755b1e04fa631, entries=150, sequenceid=356, filesize=12.0 K 2024-11-23T05:41:26,785 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/f393ea911f6e4461b4605e2f21a585c6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f393ea911f6e4461b4605e2f21a585c6 2024-11-23T05:41:26,788 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f393ea911f6e4461b4605e2f21a585c6, entries=150, sequenceid=356, filesize=12.0 K 2024-11-23T05:41:26,789 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=0 B/0 for 52e4a67ae2cd3aaf02799cd9a79db77f in 858ms, sequenceid=356, compaction requested=true 2024-11-23T05:41:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:26,789 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=83}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=83 2024-11-23T05:41:26,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=83 2024-11-23T05:41:26,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=83, resume processing ppid=82 2024-11-23T05:41:26,792 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=83, ppid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0810 sec 2024-11-23T05:41:26,794 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=82, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=82, table=TestAcidGuarantees in 2.0860 sec 2024-11-23T05:41:26,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=82 2024-11-23T05:41:26,856 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:26,857 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:26,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees 2024-11-23T05:41:26,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T05:41:26,902 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:26,903 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=84, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:26,903 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=85, ppid=84, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:26,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:26,909 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:26,909 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:41:26,913 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/51bdc8731b1a452fba674fe63b0cd970 is 50, key is test_row_0/A:col10/1732340486908/Put/seqid=0 2024-11-23T05:41:26,916 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742147_1323 (size=12301) 2024-11-23T05:41:26,941 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340546936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,941 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:26,941 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:26,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340546938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:26,942 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:26,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 198 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340546938, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,942 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:26,944 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:26,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340546941, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:26,945 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T05:41:27,047 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,047 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340547046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340547046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,048 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,048 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 199 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340547046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,048 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,056 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:27,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,056 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,059 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,059 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340547056, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,059 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,209 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,209 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T05:41:27,260 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340547259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 200 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340547259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,261 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340547259, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,262 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,267 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340547266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,267 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,268 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,317 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/51bdc8731b1a452fba674fe63b0cd970 2024-11-23T05:41:27,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/232bf2f3dae24462bcaa1b3ea50ef383 is 50, key is test_row_0/B:col10/1732340486908/Put/seqid=0 2024-11-23T05:41:27,326 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742148_1324 (size=12301) 2024-11-23T05:41:27,361 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:27,361 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,362 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,362 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,362 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,514 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:27,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,515 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T05:41:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340547566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 201 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340547566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,568 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,568 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340547566, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,569 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,577 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:27,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340547576, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:27,578 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:27,667 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,667 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,667 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,727 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/232bf2f3dae24462bcaa1b3ea50ef383 2024-11-23T05:41:27,734 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1a65aff372614b86a7c704ab389e68d2 is 50, key is test_row_0/C:col10/1732340486908/Put/seqid=0 2024-11-23T05:41:27,736 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742149_1325 (size=12301) 2024-11-23T05:41:27,820 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,820 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,820 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,972 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:27,973 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:27,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:28,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T05:41:28,079 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:28,079 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 202 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34188 deadline: 1732340548076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:28,079 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34262 deadline: 1732340548076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:28,080 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34204 deadline: 1732340548076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,080 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:28,089 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:28,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:34214 deadline: 1732340548085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:28,090 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:41:28,125 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:28,125 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] handler.RSProcedureHandler(58): pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:28,125 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=85 java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:28,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=85 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:28,137 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=367 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1a65aff372614b86a7c704ab389e68d2 2024-11-23T05:41:28,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/51bdc8731b1a452fba674fe63b0cd970 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/51bdc8731b1a452fba674fe63b0cd970 2024-11-23T05:41:28,143 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/51bdc8731b1a452fba674fe63b0cd970, entries=150, sequenceid=367, filesize=12.0 K 2024-11-23T05:41:28,144 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/232bf2f3dae24462bcaa1b3ea50ef383 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/232bf2f3dae24462bcaa1b3ea50ef383 2024-11-23T05:41:28,147 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/232bf2f3dae24462bcaa1b3ea50ef383, entries=150, sequenceid=367, filesize=12.0 K 2024-11-23T05:41:28,148 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/1a65aff372614b86a7c704ab389e68d2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1a65aff372614b86a7c704ab389e68d2 2024-11-23T05:41:28,151 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1a65aff372614b86a7c704ab389e68d2, entries=150, sequenceid=367, filesize=12.0 K 2024-11-23T05:41:28,152 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1243ms, sequenceid=367, compaction requested=true 2024-11-23T05:41:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:28,152 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:28,152 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 52e4a67ae2cd3aaf02799cd9a79db77f:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:28,152 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:28,153 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:28,153 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 52394 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:28,153 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/B is initiating minor compaction (all files) 2024-11-23T05:41:28,153 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/A is initiating minor compaction (all files) 2024-11-23T05:41:28,153 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/A in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:28,153 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/B in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:28,153 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c2a6bf0c06ed443494b07a417bfa07ee, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ab82db1fa4914dc381cf6cd2841cc99e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/2d6845fadfb34a8287802fdc3cd96df6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/51bdc8731b1a452fba674fe63b0cd970] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=51.2 K 2024-11-23T05:41:28,153 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e8dd21d325984033ba1c4e72831013a2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/9bf20008675a48adba519bb86c866543, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e494547dcb8b49a2804755b1e04fa631, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/232bf2f3dae24462bcaa1b3ea50ef383] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=48.8 K 2024-11-23T05:41:28,153 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c2a6bf0c06ed443494b07a417bfa07ee, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340482348 2024-11-23T05:41:28,153 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e8dd21d325984033ba1c4e72831013a2, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340482348 2024-11-23T05:41:28,154 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab82db1fa4914dc381cf6cd2841cc99e, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732340483506 2024-11-23T05:41:28,154 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2d6845fadfb34a8287802fdc3cd96df6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732340484699 2024-11-23T05:41:28,154 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9bf20008675a48adba519bb86c866543, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732340483506 2024-11-23T05:41:28,154 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 51bdc8731b1a452fba674fe63b0cd970, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732340486906 2024-11-23T05:41:28,154 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e494547dcb8b49a2804755b1e04fa631, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732340484699 2024-11-23T05:41:28,154 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 232bf2f3dae24462bcaa1b3ea50ef383, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732340486906 2024-11-23T05:41:28,176 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#B#compaction#279 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:28,177 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#A#compaction#278 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:28,177 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/da9ae6368ff54e06997e26c6f14a0bbb is 50, key is test_row_0/B:col10/1732340486908/Put/seqid=0 2024-11-23T05:41:28,177 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/0685489b80884eb7aa81dc26f5434807 is 50, key is test_row_0/A:col10/1732340486908/Put/seqid=0 2024-11-23T05:41:28,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742151_1327 (size=13187) 2024-11-23T05:41:28,188 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742150_1326 (size=13187) 2024-11-23T05:41:28,278 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=85 2024-11-23T05:41:28,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:28,278 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:41:28,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/1a3049ca887946e3bf5a17d499bc78cb is 50, key is test_row_0/A:col10/1732340486931/Put/seqid=0 2024-11-23T05:41:28,286 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742152_1328 (size=12301) 2024-11-23T05:41:28,584 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/0685489b80884eb7aa81dc26f5434807 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/0685489b80884eb7aa81dc26f5434807 2024-11-23T05:41:28,589 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/A of 52e4a67ae2cd3aaf02799cd9a79db77f into 0685489b80884eb7aa81dc26f5434807(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:28,589 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:28,589 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/A, priority=12, startTime=1732340488152; duration=0sec 2024-11-23T05:41:28,589 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:28,589 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:A 2024-11-23T05:41:28,589 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:28,590 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49954 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:28,591 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 52e4a67ae2cd3aaf02799cd9a79db77f/C is initiating minor compaction (all files) 2024-11-23T05:41:28,591 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 52e4a67ae2cd3aaf02799cd9a79db77f/C in TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:28,591 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6b3b2834ca54492c98518711fcc53f42, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/070e17870ed64aa6a286c8a45a56f755, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f393ea911f6e4461b4605e2f21a585c6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1a65aff372614b86a7c704ab389e68d2] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp, totalSize=48.8 K 2024-11-23T05:41:28,592 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6b3b2834ca54492c98518711fcc53f42, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=317, earliestPutTs=1732340482348 2024-11-23T05:41:28,594 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 070e17870ed64aa6a286c8a45a56f755, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=330, earliestPutTs=1732340483506 2024-11-23T05:41:28,594 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/da9ae6368ff54e06997e26c6f14a0bbb as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/da9ae6368ff54e06997e26c6f14a0bbb 2024-11-23T05:41:28,594 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f393ea911f6e4461b4605e2f21a585c6, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=356, earliestPutTs=1732340484699 2024-11-23T05:41:28,595 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1a65aff372614b86a7c704ab389e68d2, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=367, earliestPutTs=1732340486906 2024-11-23T05:41:28,599 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/B of 52e4a67ae2cd3aaf02799cd9a79db77f into da9ae6368ff54e06997e26c6f14a0bbb(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:28,599 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:28,599 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/B, priority=12, startTime=1732340488152; duration=0sec 2024-11-23T05:41:28,599 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:28,599 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:B 2024-11-23T05:41:28,604 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 52e4a67ae2cd3aaf02799cd9a79db77f#C#compaction#281 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:28,604 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/991c0ba4c4894047adff725faac899fe is 50, key is test_row_0/C:col10/1732340486908/Put/seqid=0 2024-11-23T05:41:28,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742153_1329 (size=13187) 2024-11-23T05:41:28,686 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/1a3049ca887946e3bf5a17d499bc78cb 2024-11-23T05:41:28,693 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/c612df34a1a04b3aa9d17125b07a7f77 is 50, key is test_row_0/B:col10/1732340486931/Put/seqid=0 2024-11-23T05:41:28,697 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742154_1330 (size=12301) 2024-11-23T05:41:28,780 INFO [Thread-1169 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1169. 2024-11-23T05:41:28,780 DEBUG [Thread-1169 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:28,780 DEBUG [Thread-1169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,780 DEBUG [Thread-1169 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,780 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:28,780 INFO [Thread-1161 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1161. 2024-11-23T05:41:28,780 DEBUG [Thread-1161 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:28,780 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,780 DEBUG [Thread-1161 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:28,781 INFO [Thread-1165 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1165. 2024-11-23T05:41:28,781 DEBUG [Thread-1165 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:28,781 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,781 DEBUG [Thread-1165 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,781 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:28,782 INFO [Thread-1167 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1167. 2024-11-23T05:41:28,782 DEBUG [Thread-1167 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:28,782 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,782 DEBUG [Thread-1167 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,782 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:28,782 INFO [Thread-1163 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1163. 2024-11-23T05:41:28,782 DEBUG [Thread-1163 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:28,782 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,783 DEBUG [Thread-1163 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:28,783 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:29,018 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/991c0ba4c4894047adff725faac899fe as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/991c0ba4c4894047adff725faac899fe 2024-11-23T05:41:29,022 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 52e4a67ae2cd3aaf02799cd9a79db77f/C of 52e4a67ae2cd3aaf02799cd9a79db77f into 991c0ba4c4894047adff725faac899fe(size=12.9 K), total size for store is 12.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:29,022 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:29,022 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., storeName=52e4a67ae2cd3aaf02799cd9a79db77f/C, priority=12, startTime=1732340488152; duration=0sec 2024-11-23T05:41:29,022 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:29,022 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 52e4a67ae2cd3aaf02799cd9a79db77f:C 2024-11-23T05:41:29,046 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T05:41:29,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:29,089 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. as already flushing 2024-11-23T05:41:29,089 INFO [Thread-1152 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1152. 2024-11-23T05:41:29,089 INFO [Thread-1154 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1154. 2024-11-23T05:41:29,089 INFO [Thread-1150 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1150. 2024-11-23T05:41:29,089 DEBUG [Thread-1154 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:29,090 DEBUG [Thread-1152 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:29,090 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,090 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,090 DEBUG [Thread-1150 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:29,090 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,090 DEBUG [Thread-1154 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,090 DEBUG [Thread-1152 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:29,090 DEBUG [Thread-1150 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:29,090 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:29,098 INFO [Thread-1158 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1158. 2024-11-23T05:41:29,098 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/c612df34a1a04b3aa9d17125b07a7f77 2024-11-23T05:41:29,098 DEBUG [Thread-1158 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:29,098 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,099 DEBUG [Thread-1158 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:29,099 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:29,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/ab12aef9219a4aff95036bb4df255bc5 is 50, key is test_row_0/C:col10/1732340486931/Put/seqid=0 2024-11-23T05:41:29,110 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742155_1331 (size=12301) 2024-11-23T05:41:29,511 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=392 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/ab12aef9219a4aff95036bb4df255bc5 2024-11-23T05:41:29,520 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/1a3049ca887946e3bf5a17d499bc78cb as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/1a3049ca887946e3bf5a17d499bc78cb 2024-11-23T05:41:29,525 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/1a3049ca887946e3bf5a17d499bc78cb, entries=150, sequenceid=392, filesize=12.0 K 2024-11-23T05:41:29,526 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/c612df34a1a04b3aa9d17125b07a7f77 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/c612df34a1a04b3aa9d17125b07a7f77 2024-11-23T05:41:29,531 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/c612df34a1a04b3aa9d17125b07a7f77, entries=150, sequenceid=392, filesize=12.0 K 2024-11-23T05:41:29,531 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/ab12aef9219a4aff95036bb4df255bc5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ab12aef9219a4aff95036bb4df255bc5 2024-11-23T05:41:29,534 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ab12aef9219a4aff95036bb4df255bc5, entries=150, sequenceid=392, filesize=12.0 K 2024-11-23T05:41:29,535 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=26.84 KB/27480 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1257ms, sequenceid=392, compaction requested=false 2024-11-23T05:41:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.HRegion(2603): Flush status journal for 52e4a67ae2cd3aaf02799cd9a79db77f: 2024-11-23T05:41:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:29,535 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=85}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=85 2024-11-23T05:41:29,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=85 2024-11-23T05:41:29,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=85, resume processing ppid=84 2024-11-23T05:41:29,537 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=85, ppid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.6330 sec 2024-11-23T05:41:29,538 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=84, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=84, table=TestAcidGuarantees in 2.6810 sec 2024-11-23T05:41:31,066 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=84 2024-11-23T05:41:31,067 INFO [RPCClient-NioEventLoopGroup-4-6 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:31,717 INFO [Thread-1156 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1156. 2024-11-23T05:41:31,717 DEBUG [Thread-1156 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:31,717 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:31,717 DEBUG [Thread-1156 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:31,717 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 46 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 153 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 18 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 53 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4118 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3970 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3970 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4134 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 3995 2024-11-23T05:41:31,718 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T05:41:31,718 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:41:31,718 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testGetAtomicity(AcidGuaranteesTestBase.java:107) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:41:31,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:31,718 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:31,718 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:31,719 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T05:41:31,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=86, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:31,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T05:41:31,722 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340491721"}]},"ts":"1732340491721"} 2024-11-23T05:41:31,723 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T05:41:31,723 INFO [PEWorker-1 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T05:41:31,724 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=87, ppid=86, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:41:31,725 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, UNASSIGN}] 2024-11-23T05:41:31,726 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, UNASSIGN 2024-11-23T05:41:31,726 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=52e4a67ae2cd3aaf02799cd9a79db77f, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:31,728 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=88, ppid=87, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, UNASSIGN because future has completed 2024-11-23T05:41:31,728 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:41:31,728 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:41:31,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T05:41:31,880 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(122): Close 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:31,880 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:41:31,880 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1722): Closing 52e4a67ae2cd3aaf02799cd9a79db77f, disabling compactions & flushes 2024-11-23T05:41:31,880 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:31,880 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:31,880 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. after waiting 0 ms 2024-11-23T05:41:31,880 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:31,881 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(2902): Flushing 52e4a67ae2cd3aaf02799cd9a79db77f 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T05:41:31,926 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/0b738c5d15af47d890a8beb0c07c7932 is 50, key is test_row_0/A:col10/1732340491716/Put/seqid=0 2024-11-23T05:41:31,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742156_1332 (size=12301) 2024-11-23T05:41:32,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T05:41:32,330 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/0b738c5d15af47d890a8beb0c07c7932 2024-11-23T05:41:32,335 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/999c9dfae1474a66a5453eaca0442b21 is 50, key is test_row_0/B:col10/1732340491716/Put/seqid=0 2024-11-23T05:41:32,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742157_1333 (size=12301) 2024-11-23T05:41:32,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T05:41:32,740 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/999c9dfae1474a66a5453eaca0442b21 2024-11-23T05:41:32,754 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/3f00061e79114ce8b65bc775ff9cd770 is 50, key is test_row_0/C:col10/1732340491716/Put/seqid=0 2024-11-23T05:41:32,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742158_1334 (size=12301) 2024-11-23T05:41:32,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T05:41:33,159 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=403 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/3f00061e79114ce8b65bc775ff9cd770 2024-11-23T05:41:33,171 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/A/0b738c5d15af47d890a8beb0c07c7932 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/0b738c5d15af47d890a8beb0c07c7932 2024-11-23T05:41:33,176 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/0b738c5d15af47d890a8beb0c07c7932, entries=150, sequenceid=403, filesize=12.0 K 2024-11-23T05:41:33,177 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/B/999c9dfae1474a66a5453eaca0442b21 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/999c9dfae1474a66a5453eaca0442b21 2024-11-23T05:41:33,182 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/999c9dfae1474a66a5453eaca0442b21, entries=150, sequenceid=403, filesize=12.0 K 2024-11-23T05:41:33,183 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/.tmp/C/3f00061e79114ce8b65bc775ff9cd770 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3f00061e79114ce8b65bc775ff9cd770 2024-11-23T05:41:33,187 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3f00061e79114ce8b65bc775ff9cd770, entries=150, sequenceid=403, filesize=12.0 K 2024-11-23T05:41:33,188 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1307ms, sequenceid=403, compaction requested=true 2024-11-23T05:41:33,189 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/732b2e2524004b4ea875af770ba4001a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c395d17abc6a4ef9ad423fcbdf18cab7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a6b9ade9d25440c4a38819d765a82dfc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/656a20671a7949298064b370f3b841d8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/21c0a712ae5b41b58ad61bbaca2927a0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/4d5b730e627d472c978161f39f690d68, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/11e48bcb309d4b98bf566c89934f2a25, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7c60b3d28aac49feb9423b2ed90fc21b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/004cfce606b44df8b548a34fb475e52f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/227cf58613d2468183dab2351e78619d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/087fb6c6435d426fb0b435796cd3f930, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/6fc7c49a033c4a8da27cb5d60a14f9c7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/b66121f4933a4860958100e80542ad4e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/58bcf9c139b54ce7a909bccb5fb00457, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/cabada5dc6964841a27774301813cf09, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7f57cef388a24b36a0944b4f75cbd2e8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/57f3b1b2015c405d96bd0fcfb3f78d78, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/856f694731df4635be3128b50a5d1ab5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/17a285802112416f909d88b145bd55c9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ba13f8e57a694167be47bed923ade4b2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a71d5f87376a4500ab6357c51baf3d1d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7b09f296a26f420ea8f6d6fdb1d1e749, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c2a6bf0c06ed443494b07a417bfa07ee, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ab82db1fa4914dc381cf6cd2841cc99e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/2d6845fadfb34a8287802fdc3cd96df6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/51bdc8731b1a452fba674fe63b0cd970] to archive 2024-11-23T05:41:33,190 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:33,193 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/732b2e2524004b4ea875af770ba4001a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/732b2e2524004b4ea875af770ba4001a 2024-11-23T05:41:33,194 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c395d17abc6a4ef9ad423fcbdf18cab7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c395d17abc6a4ef9ad423fcbdf18cab7 2024-11-23T05:41:33,196 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a6b9ade9d25440c4a38819d765a82dfc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a6b9ade9d25440c4a38819d765a82dfc 2024-11-23T05:41:33,198 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/656a20671a7949298064b370f3b841d8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/656a20671a7949298064b370f3b841d8 2024-11-23T05:41:33,199 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/21c0a712ae5b41b58ad61bbaca2927a0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/21c0a712ae5b41b58ad61bbaca2927a0 2024-11-23T05:41:33,201 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/4d5b730e627d472c978161f39f690d68 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/4d5b730e627d472c978161f39f690d68 2024-11-23T05:41:33,202 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/11e48bcb309d4b98bf566c89934f2a25 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/11e48bcb309d4b98bf566c89934f2a25 2024-11-23T05:41:33,204 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7c60b3d28aac49feb9423b2ed90fc21b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7c60b3d28aac49feb9423b2ed90fc21b 2024-11-23T05:41:33,206 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/004cfce606b44df8b548a34fb475e52f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/004cfce606b44df8b548a34fb475e52f 2024-11-23T05:41:33,207 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/227cf58613d2468183dab2351e78619d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/227cf58613d2468183dab2351e78619d 2024-11-23T05:41:33,209 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/087fb6c6435d426fb0b435796cd3f930 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/087fb6c6435d426fb0b435796cd3f930 2024-11-23T05:41:33,210 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/6fc7c49a033c4a8da27cb5d60a14f9c7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/6fc7c49a033c4a8da27cb5d60a14f9c7 2024-11-23T05:41:33,212 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/b66121f4933a4860958100e80542ad4e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/b66121f4933a4860958100e80542ad4e 2024-11-23T05:41:33,213 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/58bcf9c139b54ce7a909bccb5fb00457 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/58bcf9c139b54ce7a909bccb5fb00457 2024-11-23T05:41:33,214 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/cabada5dc6964841a27774301813cf09 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/cabada5dc6964841a27774301813cf09 2024-11-23T05:41:33,216 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7f57cef388a24b36a0944b4f75cbd2e8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7f57cef388a24b36a0944b4f75cbd2e8 2024-11-23T05:41:33,217 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/57f3b1b2015c405d96bd0fcfb3f78d78 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/57f3b1b2015c405d96bd0fcfb3f78d78 2024-11-23T05:41:33,219 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/856f694731df4635be3128b50a5d1ab5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/856f694731df4635be3128b50a5d1ab5 2024-11-23T05:41:33,220 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/17a285802112416f909d88b145bd55c9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/17a285802112416f909d88b145bd55c9 2024-11-23T05:41:33,221 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ba13f8e57a694167be47bed923ade4b2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ba13f8e57a694167be47bed923ade4b2 2024-11-23T05:41:33,223 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a71d5f87376a4500ab6357c51baf3d1d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/a71d5f87376a4500ab6357c51baf3d1d 2024-11-23T05:41:33,224 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7b09f296a26f420ea8f6d6fdb1d1e749 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/7b09f296a26f420ea8f6d6fdb1d1e749 2024-11-23T05:41:33,225 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c2a6bf0c06ed443494b07a417bfa07ee to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/c2a6bf0c06ed443494b07a417bfa07ee 2024-11-23T05:41:33,227 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ab82db1fa4914dc381cf6cd2841cc99e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/ab82db1fa4914dc381cf6cd2841cc99e 2024-11-23T05:41:33,228 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/2d6845fadfb34a8287802fdc3cd96df6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/2d6845fadfb34a8287802fdc3cd96df6 2024-11-23T05:41:33,229 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/51bdc8731b1a452fba674fe63b0cd970 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/51bdc8731b1a452fba674fe63b0cd970 2024-11-23T05:41:33,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/2c83edc37bca4acb801a85628c26af24, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/57fd4ef929b54373b9e7b1e533e394de, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/0c53a4605ede4e97a48bbbe07050458e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9d0f554b5e4b44a286d9c25f2a6c1a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/edf7e48ae4ed4301821f1d45cfadda09, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3b6d952ba1f3452e87eca5b07a795d7c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9b60306a7443c49392d1ba857a10d1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/67c1edcd1cf14fcdb861435ead37224f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/b1e07ce221124afeb89a0acd92e95eb8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/caf33393a6f3457088e31fb928852d66, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/a08b80cca9c646d5a3f4bc71d8148e71, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/49108fa875334e8e951db3cb7c6c5b62, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/f762bbf59f864482beea2411108a0654, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/ea7e0fbe62be49f4bc127d7c31ac8cff, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/062384a265f2411d82bfaf17f7c130ac, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/37a3d816c31b4a2582ec179611dd0df5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/65393c19ad294ab383a9bd0bc607b8de, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/358ef23715ae4618a08463720706e0ae, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3d7f9b7a815f437b92bc3da0168d4125, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/69f0967d45484197b020d6f8f71087a4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/590e53ce773c41f6b69d02af4b57e2d2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e8dd21d325984033ba1c4e72831013a2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/eb25db8f381741688d38c5c223dfecbd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/9bf20008675a48adba519bb86c866543, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e494547dcb8b49a2804755b1e04fa631, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/232bf2f3dae24462bcaa1b3ea50ef383] to archive 2024-11-23T05:41:33,231 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:33,233 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/2c83edc37bca4acb801a85628c26af24 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/2c83edc37bca4acb801a85628c26af24 2024-11-23T05:41:33,234 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/57fd4ef929b54373b9e7b1e533e394de to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/57fd4ef929b54373b9e7b1e533e394de 2024-11-23T05:41:33,235 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/0c53a4605ede4e97a48bbbe07050458e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/0c53a4605ede4e97a48bbbe07050458e 2024-11-23T05:41:33,236 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9d0f554b5e4b44a286d9c25f2a6c1a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9d0f554b5e4b44a286d9c25f2a6c1a 2024-11-23T05:41:33,237 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/edf7e48ae4ed4301821f1d45cfadda09 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/edf7e48ae4ed4301821f1d45cfadda09 2024-11-23T05:41:33,238 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3b6d952ba1f3452e87eca5b07a795d7c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3b6d952ba1f3452e87eca5b07a795d7c 2024-11-23T05:41:33,240 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9b60306a7443c49392d1ba857a10d1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/aa9b60306a7443c49392d1ba857a10d1 2024-11-23T05:41:33,241 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/67c1edcd1cf14fcdb861435ead37224f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/67c1edcd1cf14fcdb861435ead37224f 2024-11-23T05:41:33,242 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/b1e07ce221124afeb89a0acd92e95eb8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/b1e07ce221124afeb89a0acd92e95eb8 2024-11-23T05:41:33,244 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/caf33393a6f3457088e31fb928852d66 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/caf33393a6f3457088e31fb928852d66 2024-11-23T05:41:33,245 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/a08b80cca9c646d5a3f4bc71d8148e71 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/a08b80cca9c646d5a3f4bc71d8148e71 2024-11-23T05:41:33,246 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/49108fa875334e8e951db3cb7c6c5b62 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/49108fa875334e8e951db3cb7c6c5b62 2024-11-23T05:41:33,247 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/f762bbf59f864482beea2411108a0654 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/f762bbf59f864482beea2411108a0654 2024-11-23T05:41:33,248 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/ea7e0fbe62be49f4bc127d7c31ac8cff to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/ea7e0fbe62be49f4bc127d7c31ac8cff 2024-11-23T05:41:33,250 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/062384a265f2411d82bfaf17f7c130ac to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/062384a265f2411d82bfaf17f7c130ac 2024-11-23T05:41:33,251 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/37a3d816c31b4a2582ec179611dd0df5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/37a3d816c31b4a2582ec179611dd0df5 2024-11-23T05:41:33,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/65393c19ad294ab383a9bd0bc607b8de to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/65393c19ad294ab383a9bd0bc607b8de 2024-11-23T05:41:33,252 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/358ef23715ae4618a08463720706e0ae to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/358ef23715ae4618a08463720706e0ae 2024-11-23T05:41:33,253 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3d7f9b7a815f437b92bc3da0168d4125 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/3d7f9b7a815f437b92bc3da0168d4125 2024-11-23T05:41:33,254 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/69f0967d45484197b020d6f8f71087a4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/69f0967d45484197b020d6f8f71087a4 2024-11-23T05:41:33,255 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/590e53ce773c41f6b69d02af4b57e2d2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/590e53ce773c41f6b69d02af4b57e2d2 2024-11-23T05:41:33,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e8dd21d325984033ba1c4e72831013a2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e8dd21d325984033ba1c4e72831013a2 2024-11-23T05:41:33,256 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/eb25db8f381741688d38c5c223dfecbd to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/eb25db8f381741688d38c5c223dfecbd 2024-11-23T05:41:33,257 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/9bf20008675a48adba519bb86c866543 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/9bf20008675a48adba519bb86c866543 2024-11-23T05:41:33,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e494547dcb8b49a2804755b1e04fa631 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/e494547dcb8b49a2804755b1e04fa631 2024-11-23T05:41:33,258 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/232bf2f3dae24462bcaa1b3ea50ef383 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/232bf2f3dae24462bcaa1b3ea50ef383 2024-11-23T05:41:33,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/9db772f9b8b54a0f9cd7a11d4028d13d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/012a8c38b56240dc840840986334aa4b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f247fc0c05494f6a81d0ebf987517c49, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1c85a252611843ca9db152345724a354, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ed8ecd0d09274e33a7b0505c5dd20451, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/c1858577e77145d8a821b36a4963849d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/17108ff909f545a6bee1eefcc35990ef, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/384de24bd27d489ba48e1abcc12a0112, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3cbd20485bae47c59724bd994a61b9f3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6036c505fe6a48219064ed5a44e8664c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/0ac3d58b4d85410abf41e58675c2aa45, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1662a7a1edfe4a38bb93c5290392df74, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a9e2ae12e12043a6ad9f09ef2a9833a5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a7f4aa147db9428f8e311eaf61cdfc0a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/b6149feee28a40478864a333fcb5bf7c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/fd5e723876da42f6ab0abb26d3400c72, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a0a818fc083c4434bf8be526f7831747, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/2ca4b9331d784e94967929411ff5ef59, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/4ca513deffcc427fa7901fc9cc89fc92, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/e58c319d37ed4d1d86166be53f9810e7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/5aad5074fdb7477d8171bbbf8143d426, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6b3b2834ca54492c98518711fcc53f42, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/26b9d67a84754bc1b1535d5ad157f71e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/070e17870ed64aa6a286c8a45a56f755, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f393ea911f6e4461b4605e2f21a585c6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1a65aff372614b86a7c704ab389e68d2] to archive 2024-11-23T05:41:33,260 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:33,261 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/9db772f9b8b54a0f9cd7a11d4028d13d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/9db772f9b8b54a0f9cd7a11d4028d13d 2024-11-23T05:41:33,263 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/012a8c38b56240dc840840986334aa4b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/012a8c38b56240dc840840986334aa4b 2024-11-23T05:41:33,264 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f247fc0c05494f6a81d0ebf987517c49 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f247fc0c05494f6a81d0ebf987517c49 2024-11-23T05:41:33,265 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1c85a252611843ca9db152345724a354 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1c85a252611843ca9db152345724a354 2024-11-23T05:41:33,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ed8ecd0d09274e33a7b0505c5dd20451 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ed8ecd0d09274e33a7b0505c5dd20451 2024-11-23T05:41:33,266 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/c1858577e77145d8a821b36a4963849d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/c1858577e77145d8a821b36a4963849d 2024-11-23T05:41:33,267 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/17108ff909f545a6bee1eefcc35990ef to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/17108ff909f545a6bee1eefcc35990ef 2024-11-23T05:41:33,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/384de24bd27d489ba48e1abcc12a0112 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/384de24bd27d489ba48e1abcc12a0112 2024-11-23T05:41:33,268 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3cbd20485bae47c59724bd994a61b9f3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3cbd20485bae47c59724bd994a61b9f3 2024-11-23T05:41:33,269 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6036c505fe6a48219064ed5a44e8664c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6036c505fe6a48219064ed5a44e8664c 2024-11-23T05:41:33,270 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/0ac3d58b4d85410abf41e58675c2aa45 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/0ac3d58b4d85410abf41e58675c2aa45 2024-11-23T05:41:33,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1662a7a1edfe4a38bb93c5290392df74 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1662a7a1edfe4a38bb93c5290392df74 2024-11-23T05:41:33,271 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a9e2ae12e12043a6ad9f09ef2a9833a5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a9e2ae12e12043a6ad9f09ef2a9833a5 2024-11-23T05:41:33,272 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a7f4aa147db9428f8e311eaf61cdfc0a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a7f4aa147db9428f8e311eaf61cdfc0a 2024-11-23T05:41:33,273 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/b6149feee28a40478864a333fcb5bf7c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/b6149feee28a40478864a333fcb5bf7c 2024-11-23T05:41:33,274 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/fd5e723876da42f6ab0abb26d3400c72 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/fd5e723876da42f6ab0abb26d3400c72 2024-11-23T05:41:33,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a0a818fc083c4434bf8be526f7831747 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/a0a818fc083c4434bf8be526f7831747 2024-11-23T05:41:33,275 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/2ca4b9331d784e94967929411ff5ef59 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/2ca4b9331d784e94967929411ff5ef59 2024-11-23T05:41:33,276 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/4ca513deffcc427fa7901fc9cc89fc92 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/4ca513deffcc427fa7901fc9cc89fc92 2024-11-23T05:41:33,277 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/e58c319d37ed4d1d86166be53f9810e7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/e58c319d37ed4d1d86166be53f9810e7 2024-11-23T05:41:33,278 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/5aad5074fdb7477d8171bbbf8143d426 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/5aad5074fdb7477d8171bbbf8143d426 2024-11-23T05:41:33,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6b3b2834ca54492c98518711fcc53f42 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/6b3b2834ca54492c98518711fcc53f42 2024-11-23T05:41:33,279 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/26b9d67a84754bc1b1535d5ad157f71e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/26b9d67a84754bc1b1535d5ad157f71e 2024-11-23T05:41:33,280 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/070e17870ed64aa6a286c8a45a56f755 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/070e17870ed64aa6a286c8a45a56f755 2024-11-23T05:41:33,281 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f393ea911f6e4461b4605e2f21a585c6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/f393ea911f6e4461b4605e2f21a585c6 2024-11-23T05:41:33,282 DEBUG [StoreCloser-TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1a65aff372614b86a7c704ab389e68d2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/1a65aff372614b86a7c704ab389e68d2 2024-11-23T05:41:33,286 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/recovered.edits/406.seqid, newMaxSeqId=406, maxSeqId=1 2024-11-23T05:41:33,286 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. 2024-11-23T05:41:33,286 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] regionserver.HRegion(1676): Region close journal for 52e4a67ae2cd3aaf02799cd9a79db77f: Waiting for close lock at 1732340491880Running coprocessor pre-close hooks at 1732340491880Disabling compacts and flushes for region at 1732340491880Disabling writes for close at 1732340491880Obtaining lock to block concurrent updates at 1732340491881 (+1 ms)Preparing flush snapshotting stores in 52e4a67ae2cd3aaf02799cd9a79db77f at 1732340491881Finished memstore snapshotting TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732340491881Flushing stores of TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f. at 1732340491882 (+1 ms)Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/A: creating writer at 1732340491882Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/A: appending metadata at 1732340491926 (+44 ms)Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/A: closing flushed file at 1732340491926Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/B: creating writer at 1732340492333 (+407 ms)Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/B: appending metadata at 1732340492335 (+2 ms)Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/B: closing flushed file at 1732340492335Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/C: creating writer at 1732340492750 (+415 ms)Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/C: appending metadata at 1732340492753 (+3 ms)Flushing 52e4a67ae2cd3aaf02799cd9a79db77f/C: closing flushed file at 1732340492753Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@63581b19: reopening flushed file at 1732340493169 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@2b1a33f4: reopening flushed file at 1732340493177 (+8 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@58a426e9: reopening flushed file at 1732340493182 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 52e4a67ae2cd3aaf02799cd9a79db77f in 1307ms, sequenceid=403, compaction requested=true at 1732340493188 (+6 ms)Writing region close event to WAL at 1732340493283 (+95 ms)Running coprocessor post-close hooks at 1732340493286 (+3 ms)Closed at 1732340493286 2024-11-23T05:41:33,288 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=89}] handler.UnassignRegionHandler(157): Closed 52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:33,288 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=88 updating hbase:meta row=52e4a67ae2cd3aaf02799cd9a79db77f, regionState=CLOSED 2024-11-23T05:41:33,290 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=89, ppid=88, state=RUNNABLE, hasLock=false; CloseRegionProcedure 52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:41:33,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=89, resume processing ppid=88 2024-11-23T05:41:33,292 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=89, ppid=88, state=SUCCESS, hasLock=false; CloseRegionProcedure 52e4a67ae2cd3aaf02799cd9a79db77f, server=9abfc7c2569e,45719,1732340394120 in 1.5620 sec 2024-11-23T05:41:33,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=88, resume processing ppid=87 2024-11-23T05:41:33,293 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=88, ppid=87, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=52e4a67ae2cd3aaf02799cd9a79db77f, UNASSIGN in 1.5670 sec 2024-11-23T05:41:33,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=87, resume processing ppid=86 2024-11-23T05:41:33,294 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=87, ppid=86, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5700 sec 2024-11-23T05:41:33,296 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340493295"}]},"ts":"1732340493295"} 2024-11-23T05:41:33,297 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T05:41:33,297 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T05:41:33,299 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=86, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5790 sec 2024-11-23T05:41:33,698 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T05:41:33,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=86 2024-11-23T05:41:33,878 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:33,879 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T05:41:33,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:33,885 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=90, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:33,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T05:41:33,887 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=90, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:33,890 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:33,893 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/recovered.edits] 2024-11-23T05:41:33,896 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/0685489b80884eb7aa81dc26f5434807 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/0685489b80884eb7aa81dc26f5434807 2024-11-23T05:41:33,897 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/0b738c5d15af47d890a8beb0c07c7932 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/0b738c5d15af47d890a8beb0c07c7932 2024-11-23T05:41:33,899 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/1a3049ca887946e3bf5a17d499bc78cb to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/A/1a3049ca887946e3bf5a17d499bc78cb 2024-11-23T05:41:33,902 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/999c9dfae1474a66a5453eaca0442b21 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/999c9dfae1474a66a5453eaca0442b21 2024-11-23T05:41:33,903 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/c612df34a1a04b3aa9d17125b07a7f77 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/c612df34a1a04b3aa9d17125b07a7f77 2024-11-23T05:41:33,904 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/da9ae6368ff54e06997e26c6f14a0bbb to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/B/da9ae6368ff54e06997e26c6f14a0bbb 2024-11-23T05:41:33,906 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3f00061e79114ce8b65bc775ff9cd770 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/3f00061e79114ce8b65bc775ff9cd770 2024-11-23T05:41:33,907 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/991c0ba4c4894047adff725faac899fe to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/991c0ba4c4894047adff725faac899fe 2024-11-23T05:41:33,908 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ab12aef9219a4aff95036bb4df255bc5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/C/ab12aef9219a4aff95036bb4df255bc5 2024-11-23T05:41:33,909 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/recovered.edits/406.seqid to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f/recovered.edits/406.seqid 2024-11-23T05:41:33,910 DEBUG [HFileArchiver-3 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/52e4a67ae2cd3aaf02799cd9a79db77f 2024-11-23T05:41:33,910 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T05:41:33,911 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=90, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:33,913 WARN [PEWorker-1 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T05:41:33,915 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T05:41:33,916 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=90, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:33,916 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T05:41:33,916 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732340493916"}]},"ts":"9223372036854775807"} 2024-11-23T05:41:33,917 INFO [PEWorker-1 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T05:41:33,917 DEBUG [PEWorker-1 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 52e4a67ae2cd3aaf02799cd9a79db77f, NAME => 'TestAcidGuarantees,,1732340466573.52e4a67ae2cd3aaf02799cd9a79db77f.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T05:41:33,918 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T05:41:33,918 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732340493918"}]},"ts":"9223372036854775807"} 2024-11-23T05:41:33,919 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T05:41:33,920 DEBUG [PEWorker-1 {}] procedure.DeleteTableProcedure(133): Finished pid=90, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:33,921 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=90, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 41 msec 2024-11-23T05:41:33,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=90 2024-11-23T05:41:33,996 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T05:41:34,000 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:34,010 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testGetAtomicity Thread=240 (was 238) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935771990_22 at /127.0.0.1:45570 [Waiting for operation #321] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-3 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-24366377_22 at /127.0.0.1:49976 [Waiting for operation #117] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=455 (was 451) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=522 (was 524), ProcessCount=11 (was 11), AvailableMemoryMB=8722 (was 8883) 2024-11-23T05:41:34,020 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=240, OpenFileDescriptor=455, MaxFileDescriptor=1048576, SystemLoadAverage=522, ProcessCount=11, AvailableMemoryMB=8721 2024-11-23T05:41:34,021 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:41:34,021 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:41:34,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:34,023 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:41:34,023 DEBUG [PEWorker-3 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:34,023 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 91 2024-11-23T05:41:34,024 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:41:34,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T05:41:34,030 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742159_1335 (size=959) 2024-11-23T05:41:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T05:41:34,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T05:41:34,432 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:41:34,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742160_1336 (size=53) 2024-11-23T05:41:34,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T05:41:34,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:41:34,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing a9d4eac7c30b3b93cecf8ac4d22b4ff7, disabling compactions & flushes 2024-11-23T05:41:34,839 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:34,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:34,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. after waiting 0 ms 2024-11-23T05:41:34,839 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:34,839 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:34,840 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: Waiting for close lock at 1732340494839Disabling compacts and flushes for region at 1732340494839Disabling writes for close at 1732340494839Writing region close event to WAL at 1732340494839Closed at 1732340494839 2024-11-23T05:41:34,841 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:41:34,842 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732340494841"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732340494841"}]},"ts":"1732340494841"} 2024-11-23T05:41:34,845 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:41:34,846 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:41:34,847 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340494846"}]},"ts":"1732340494846"} 2024-11-23T05:41:34,849 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T05:41:34,849 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, ASSIGN}] 2024-11-23T05:41:34,850 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, ASSIGN 2024-11-23T05:41:34,851 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, ASSIGN; state=OFFLINE, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=false 2024-11-23T05:41:35,002 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:35,006 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=92, ppid=91, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, ASSIGN because future has completed 2024-11-23T05:41:35,006 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:41:35,156 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T05:41:35,163 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:35,163 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7752): Opening region: {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:41:35,163 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,163 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:41:35,164 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7794): checking encryption for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,164 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(7797): checking classloading for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,166 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,168 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4eac7c30b3b93cecf8ac4d22b4ff7 columnFamilyName A 2024-11-23T05:41:35,168 DEBUG [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:35,169 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(327): Store=a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:35,169 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,171 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4eac7c30b3b93cecf8ac4d22b4ff7 columnFamilyName B 2024-11-23T05:41:35,171 DEBUG [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:35,172 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(327): Store=a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:35,172 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,174 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4eac7c30b3b93cecf8ac4d22b4ff7 columnFamilyName C 2024-11-23T05:41:35,174 DEBUG [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:35,175 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(327): Store=a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:35,175 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1038): replaying wal for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,176 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,177 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,179 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1048): stopping wal replay for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,179 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1060): Cleaning up temporary data for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,180 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:41:35,181 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1093): writing seq id for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,184 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:41:35,184 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1114): Opened a9d4eac7c30b3b93cecf8ac4d22b4ff7; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=64286393, jitterRate=-0.04205809533596039}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:41:35,185 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:35,185 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegion(1006): Region open journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: Running coprocessor pre-open hook at 1732340495164Writing region info on filesystem at 1732340495164Initializing all the Stores at 1732340495165 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340495165Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340495165Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340495166 (+1 ms)Cleaning up temporary data from old regions at 1732340495179 (+13 ms)Running coprocessor post-open hooks at 1732340495185 (+6 ms)Region opened successfully at 1732340495185 2024-11-23T05:41:35,186 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., pid=93, masterSystemTime=1732340495158 2024-11-23T05:41:35,188 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:35,188 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=93}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:35,188 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=92 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:35,189 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=93, ppid=92, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:41:35,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=93, resume processing ppid=92 2024-11-23T05:41:35,192 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=93, ppid=92, state=SUCCESS, hasLock=false; OpenRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 in 184 msec 2024-11-23T05:41:35,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=92, resume processing ppid=91 2024-11-23T05:41:35,194 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=92, ppid=91, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, ASSIGN in 343 msec 2024-11-23T05:41:35,194 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:41:35,195 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340495194"}]},"ts":"1732340495194"} 2024-11-23T05:41:35,196 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T05:41:35,197 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=91, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:41:35,198 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=91, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1760 sec 2024-11-23T05:41:36,167 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=91 2024-11-23T05:41:36,168 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:36,171 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4dda73f8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:36,171 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:36,171 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:36,173 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:36,173 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@173cbcc6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:36,174 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:36,175 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35178, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:36,176 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3e804398, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:36,176 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:36,177 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:36,178 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:36,179 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41602, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:36,180 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:41:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,40189,1732340393249 2024-11-23T05:41:36,180 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@5a3b8a6f 2024-11-23T05:41:36,181 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:41:36,182 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35186, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:41:36,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:41:36,183 INFO [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:41:36,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=94, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:36,196 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742161_1337 (size=995) 2024-11-23T05:41:36,598 DEBUG [PEWorker-4 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T05:41:36,599 INFO [PEWorker-4 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T05:41:36,602 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=95, ppid=94, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:41:36,606 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, REOPEN/MOVE}] 2024-11-23T05:41:36,607 INFO [PEWorker-5 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, REOPEN/MOVE 2024-11-23T05:41:36,608 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:36,611 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, REOPEN/MOVE because future has completed 2024-11-23T05:41:36,611 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:41:36,611 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:41:36,766 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(122): Close a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:36,766 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:41:36,766 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1722): Closing a9d4eac7c30b3b93cecf8ac4d22b4ff7, disabling compactions & flushes 2024-11-23T05:41:36,766 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:36,766 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:36,766 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. after waiting 0 ms 2024-11-23T05:41:36,767 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:36,775 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T05:41:36,777 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:36,777 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegion(1676): Region close journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: Waiting for close lock at 1732340496766Running coprocessor pre-close hooks at 1732340496766Disabling compacts and flushes for region at 1732340496766Disabling writes for close at 1732340496767 (+1 ms)Writing region close event to WAL at 1732340496768 (+1 ms)Running coprocessor post-close hooks at 1732340496777 (+9 ms)Closed at 1732340496777 2024-11-23T05:41:36,777 WARN [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] regionserver.HRegionServer(3299): Not adding moved region record: a9d4eac7c30b3b93cecf8ac4d22b4ff7 to self. 2024-11-23T05:41:36,779 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=97}] handler.UnassignRegionHandler(157): Closed a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:36,780 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=CLOSED 2024-11-23T05:41:36,781 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=97, ppid=96, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:41:36,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=97, resume processing ppid=96 2024-11-23T05:41:36,783 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=97, ppid=96, state=SUCCESS, hasLock=false; CloseRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 in 170 msec 2024-11-23T05:41:36,783 INFO [PEWorker-5 {}] assignment.TransitRegionStateProcedure(269): Starting pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, REOPEN/MOVE; state=CLOSED, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=true 2024-11-23T05:41:36,934 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:36,942 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=96, ppid=95, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, REOPEN/MOVE because future has completed 2024-11-23T05:41:36,942 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:41:37,101 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,102 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7752): Opening region: {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:41:37,102 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,103 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:41:37,103 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7794): checking encryption for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,103 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(7797): checking classloading for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,106 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,107 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4eac7c30b3b93cecf8ac4d22b4ff7 columnFamilyName A 2024-11-23T05:41:37,108 DEBUG [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:37,109 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(327): Store=a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:37,109 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,110 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4eac7c30b3b93cecf8ac4d22b4ff7 columnFamilyName B 2024-11-23T05:41:37,110 DEBUG [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:37,111 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(327): Store=a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:37,111 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,112 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region a9d4eac7c30b3b93cecf8ac4d22b4ff7 columnFamilyName C 2024-11-23T05:41:37,112 DEBUG [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:37,112 INFO [StoreOpener-a9d4eac7c30b3b93cecf8ac4d22b4ff7-1 {}] regionserver.HStore(327): Store=a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:41:37,112 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1038): replaying wal for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,113 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,114 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,115 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1048): stopping wal replay for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,115 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1060): Cleaning up temporary data for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,116 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:41:37,117 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1093): writing seq id for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,118 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1114): Opened a9d4eac7c30b3b93cecf8ac4d22b4ff7; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61856685, jitterRate=-0.07826356589794159}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:41:37,118 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1122): Running coprocessor post-open hooks for a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,119 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegion(1006): Region open journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: Running coprocessor pre-open hook at 1732340497103Writing region info on filesystem at 1732340497103Initializing all the Stores at 1732340497105 (+2 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340497105Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340497105Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340497106 (+1 ms)Cleaning up temporary data from old regions at 1732340497115 (+9 ms)Running coprocessor post-open hooks at 1732340497118 (+3 ms)Region opened successfully at 1732340497118 2024-11-23T05:41:37,119 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., pid=98, masterSystemTime=1732340497096 2024-11-23T05:41:37,121 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,121 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=98}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,121 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=96 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=OPEN, openSeqNum=5, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,123 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=98, ppid=96, state=RUNNABLE, hasLock=false; OpenRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:41:37,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=98, resume processing ppid=96 2024-11-23T05:41:37,125 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=98, ppid=96, state=SUCCESS, hasLock=false; OpenRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 in 182 msec 2024-11-23T05:41:37,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=96, resume processing ppid=95 2024-11-23T05:41:37,127 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=96, ppid=95, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, REOPEN/MOVE in 519 msec 2024-11-23T05:41:37,129 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=95, resume processing ppid=94 2024-11-23T05:41:37,130 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=95, ppid=94, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 526 msec 2024-11-23T05:41:37,131 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=94, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 946 msec 2024-11-23T05:41:37,132 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=94 2024-11-23T05:41:37,132 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:37,133 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@554bdeef, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,133 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,133 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,133 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,133 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@55732bb8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,134 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,134 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35212, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,135 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@13d8aa16, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,135 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5232d48c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,135 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,135 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,136 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@42c00bf, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,136 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,137 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35226, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,137 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@65a06781, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,138 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4b67a6ca, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,138 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,138 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,138 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@310cbb31, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,139 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,140 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35250, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,140 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@50417be8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,140 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@12688331, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,140 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,141 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,141 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2123ee6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,141 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,142 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35272, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,142 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@22888ccd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,143 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1456e4e8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,143 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,143 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,143 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3dd4e677, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,144 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35298, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,145 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ef38443, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,145 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@74436e44, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,145 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,145 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,146 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@419c85f1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,146 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,147 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,147 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35324, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,148 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3ef525db, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,148 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@15099ff4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,148 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,148 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,149 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@43f7b26c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,149 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,150 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35348, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,151 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@164ff8dd, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,151 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2cbfebb0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,151 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,151 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,152 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@190627a2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,152 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,153 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,153 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35364, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,154 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5bc5780f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,154 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17bdde2c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,154 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,154 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,155 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@593e0d26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,155 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,156 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,157 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35374, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,157 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1b57273f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,158 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7608485b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,158 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:41:37,158 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:41:37,159 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:41:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:41:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:41:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@60788bd6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:41:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:41:37,159 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:37,160 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:35384, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:41:37,161 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@137e8a84, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:41:37,161 DEBUG [Thread-1548 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,161 DEBUG [Thread-1544 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,161 DEBUG [Thread-1553 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,162 DEBUG [Thread-1559 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,162 DEBUG [Thread-1557 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,162 DEBUG [Thread-1542 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,162 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,162 DEBUG [Thread-1548 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,163 DEBUG [Thread-1551 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,163 DEBUG [Thread-1559 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,163 DEBUG [Thread-1548 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,163 DEBUG [Thread-1555 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41608, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41604, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,164 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41610, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,164 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,163 DEBUG [Thread-1559 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,164 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,165 DEBUG [Thread-1540 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,163 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,165 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41634, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,165 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,166 DEBUG [Thread-1546 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:41:37,166 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41648, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,166 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,166 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees 2024-11-23T05:41:37,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41664, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:41:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:41:37,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41636, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,167 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,167 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,167 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:37,168 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,168 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41698, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,169 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:41692, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:41:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,170 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T05:41:37,170 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,171 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=99, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,171 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=100, ppid=99, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:37,171 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,172 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:41:37,179 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,179 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:37,179 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:41:37,224 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123639465f9c84b47d79fbeaf1f4485d301_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340497173/Put/seqid=0 2024-11-23T05:41:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340557218, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,227 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340557222, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,227 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,229 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340557223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,232 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340557223, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,233 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340557224, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,233 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,236 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742162_1338 (size=14594) 2024-11-23T05:41:37,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T05:41:37,324 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:37,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:37,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,325 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,325 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,326 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,337 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340557336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,337 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340557336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340557336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,340 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,340 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340557336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,340 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,341 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,341 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340557336, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,341 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,478 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:37,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:37,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,478 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,478 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T05:41:37,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340557546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,549 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340557546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,549 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340557546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340557546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,551 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,551 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340557546, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,552 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,631 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:37,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:37,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,631 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,636 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:37,639 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123639465f9c84b47d79fbeaf1f4485d301_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123639465f9c84b47d79fbeaf1f4485d301_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:37,640 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c3ce8d689ec440cabceb6e94d4ec7540, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:37,641 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c3ce8d689ec440cabceb6e94d4ec7540 is 175, key is test_row_0/A:col10/1732340497173/Put/seqid=0 2024-11-23T05:41:37,643 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742163_1339 (size=39545) 2024-11-23T05:41:37,783 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,784 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,784 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T05:41:37,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340557855, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340557856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340557856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,860 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,860 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340557856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,861 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340557856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:37,861 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:37,936 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:37,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,936 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:37,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:37,937 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,937 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:37,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,044 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=15, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c3ce8d689ec440cabceb6e94d4ec7540 2024-11-23T05:41:38,063 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/eed7c0199cb44e9aad48f3ad4a31df41 is 50, key is test_row_0/B:col10/1732340497173/Put/seqid=0 2024-11-23T05:41:38,067 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742164_1340 (size=9657) 2024-11-23T05:41:38,089 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:38,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:38,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:38,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:38,089 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,089 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,090 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,242 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:38,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:38,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:38,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:38,242 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,242 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,243 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,306 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T05:41:38,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340558366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:38,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340558366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,369 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:38,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:38,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340558366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:38,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340558366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:38,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340558366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:38,370 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:38,394 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:38,394 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:38,395 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] handler.RSProcedureHandler(58): pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,395 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=100 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=100 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:38,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/eed7c0199cb44e9aad48f3ad4a31df41 2024-11-23T05:41:38,489 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/3b400278b0444cf98406c79d80907ffc is 50, key is test_row_0/C:col10/1732340497173/Put/seqid=0 2024-11-23T05:41:38,492 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742165_1341 (size=9657) 2024-11-23T05:41:38,494 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=15 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/3b400278b0444cf98406c79d80907ffc 2024-11-23T05:41:38,497 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c3ce8d689ec440cabceb6e94d4ec7540 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c3ce8d689ec440cabceb6e94d4ec7540 2024-11-23T05:41:38,502 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c3ce8d689ec440cabceb6e94d4ec7540, entries=200, sequenceid=15, filesize=38.6 K 2024-11-23T05:41:38,502 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/eed7c0199cb44e9aad48f3ad4a31df41 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/eed7c0199cb44e9aad48f3ad4a31df41 2024-11-23T05:41:38,506 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/eed7c0199cb44e9aad48f3ad4a31df41, entries=100, sequenceid=15, filesize=9.4 K 2024-11-23T05:41:38,506 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/3b400278b0444cf98406c79d80907ffc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3b400278b0444cf98406c79d80907ffc 2024-11-23T05:41:38,510 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3b400278b0444cf98406c79d80907ffc, entries=100, sequenceid=15, filesize=9.4 K 2024-11-23T05:41:38,510 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1331ms, sequenceid=15, compaction requested=false 2024-11-23T05:41:38,511 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:38,547 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=100 2024-11-23T05:41:38,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:38,547 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:41:38,554 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eef412bc42f64a068026f6c03b80d1b8_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340497207/Put/seqid=0 2024-11-23T05:41:38,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742166_1342 (size=12154) 2024-11-23T05:41:38,564 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:38,567 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123eef412bc42f64a068026f6c03b80d1b8_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eef412bc42f64a068026f6c03b80d1b8_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:38,569 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/09e77500c9604211b89ef0af56a3cabb, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:38,570 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/09e77500c9604211b89ef0af56a3cabb is 175, key is test_row_0/A:col10/1732340497207/Put/seqid=0 2024-11-23T05:41:38,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742167_1343 (size=30955) 2024-11-23T05:41:38,813 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:41:38,973 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/09e77500c9604211b89ef0af56a3cabb 2024-11-23T05:41:38,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/0cabf826ea0041c9ac656f88b8204d73 is 50, key is test_row_0/B:col10/1732340497207/Put/seqid=0 2024-11-23T05:41:38,982 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742168_1344 (size=12001) 2024-11-23T05:41:38,987 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/0cabf826ea0041c9ac656f88b8204d73 2024-11-23T05:41:38,993 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/1b7d4d104e9e4dd8a787df4319ae87c7 is 50, key is test_row_0/C:col10/1732340497207/Put/seqid=0 2024-11-23T05:41:39,002 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742169_1345 (size=12001) 2024-11-23T05:41:39,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T05:41:39,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:39,379 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:39,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340559386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,392 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340559389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,392 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,392 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2169 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,393 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340559389, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340559390, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,397 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340559392, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,397 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,398 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,403 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/1b7d4d104e9e4dd8a787df4319ae87c7 2024-11-23T05:41:39,406 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/09e77500c9604211b89ef0af56a3cabb as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/09e77500c9604211b89ef0af56a3cabb 2024-11-23T05:41:39,409 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/09e77500c9604211b89ef0af56a3cabb, entries=150, sequenceid=40, filesize=30.2 K 2024-11-23T05:41:39,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/0cabf826ea0041c9ac656f88b8204d73 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/0cabf826ea0041c9ac656f88b8204d73 2024-11-23T05:41:39,412 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/0cabf826ea0041c9ac656f88b8204d73, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T05:41:39,413 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/1b7d4d104e9e4dd8a787df4319ae87c7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1b7d4d104e9e4dd8a787df4319ae87c7 2024-11-23T05:41:39,417 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1b7d4d104e9e4dd8a787df4319ae87c7, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T05:41:39,418 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 871ms, sequenceid=40, compaction requested=false 2024-11-23T05:41:39,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:39,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:39,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=100}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=100 2024-11-23T05:41:39,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=100 2024-11-23T05:41:39,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=100, resume processing ppid=99 2024-11-23T05:41:39,420 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=100, ppid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.2480 sec 2024-11-23T05:41:39,422 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=99, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=99, table=TestAcidGuarantees in 2.2560 sec 2024-11-23T05:41:39,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:39,502 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:39,502 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:41:39,509 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ff836a45fa4541859402cf461005dc0d_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340499383/Put/seqid=0 2024-11-23T05:41:39,512 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742170_1346 (size=14594) 2024-11-23T05:41:39,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340559542, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340559543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,552 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340559549, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,552 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,553 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,558 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340559550, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,559 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340559655, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340559656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340559656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,662 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340559666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,671 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340559866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340559866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,871 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,871 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340559867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,872 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,881 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:39,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340559876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:39,882 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:39,913 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:39,916 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ff836a45fa4541859402cf461005dc0d_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ff836a45fa4541859402cf461005dc0d_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:39,917 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/4b964a1f09a74979abce8e822152e3e8, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:39,918 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/4b964a1f09a74979abce8e822152e3e8 is 175, key is test_row_0/A:col10/1732340499383/Put/seqid=0 2024-11-23T05:41:39,921 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742171_1347 (size=39549) 2024-11-23T05:41:40,183 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340560176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340560176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,184 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,184 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340560176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,184 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,185 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,193 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340560186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,194 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,321 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/4b964a1f09a74979abce8e822152e3e8 2024-11-23T05:41:40,328 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/9f81c4f8f79b436f9ca8e0dfc5721174 is 50, key is test_row_0/B:col10/1732340499383/Put/seqid=0 2024-11-23T05:41:40,331 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742172_1348 (size=12001) 2024-11-23T05:41:40,335 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/9f81c4f8f79b436f9ca8e0dfc5721174 2024-11-23T05:41:40,342 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/ee7ca6edbba24a788e18b20c8bc5e663 is 50, key is test_row_0/C:col10/1732340499383/Put/seqid=0 2024-11-23T05:41:40,346 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742173_1349 (size=12001) 2024-11-23T05:41:40,702 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340560696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,703 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340560696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,703 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,707 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340560696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,707 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:40,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340560708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:40,714 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:40,746 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/ee7ca6edbba24a788e18b20c8bc5e663 2024-11-23T05:41:40,750 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/4b964a1f09a74979abce8e822152e3e8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/4b964a1f09a74979abce8e822152e3e8 2024-11-23T05:41:40,753 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/4b964a1f09a74979abce8e822152e3e8, entries=200, sequenceid=52, filesize=38.6 K 2024-11-23T05:41:40,754 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/9f81c4f8f79b436f9ca8e0dfc5721174 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/9f81c4f8f79b436f9ca8e0dfc5721174 2024-11-23T05:41:40,758 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/9f81c4f8f79b436f9ca8e0dfc5721174, entries=150, sequenceid=52, filesize=11.7 K 2024-11-23T05:41:40,759 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/ee7ca6edbba24a788e18b20c8bc5e663 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ee7ca6edbba24a788e18b20c8bc5e663 2024-11-23T05:41:40,762 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ee7ca6edbba24a788e18b20c8bc5e663, entries=150, sequenceid=52, filesize=11.7 K 2024-11-23T05:41:40,763 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1261ms, sequenceid=52, compaction requested=true 2024-11-23T05:41:40,763 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:40,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:40,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:40,763 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:40,763 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:40,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:40,763 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:40,764 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:40,764 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:40,764 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 110049 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:40,764 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/A is initiating minor compaction (all files) 2024-11-23T05:41:40,764 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/B is initiating minor compaction (all files) 2024-11-23T05:41:40,764 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/B in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:40,764 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/A in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:40,764 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/eed7c0199cb44e9aad48f3ad4a31df41, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/0cabf826ea0041c9ac656f88b8204d73, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/9f81c4f8f79b436f9ca8e0dfc5721174] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=32.9 K 2024-11-23T05:41:40,764 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c3ce8d689ec440cabceb6e94d4ec7540, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/09e77500c9604211b89ef0af56a3cabb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/4b964a1f09a74979abce8e822152e3e8] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=107.5 K 2024-11-23T05:41:40,765 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:40,765 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c3ce8d689ec440cabceb6e94d4ec7540, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/09e77500c9604211b89ef0af56a3cabb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/4b964a1f09a74979abce8e822152e3e8] 2024-11-23T05:41:40,765 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting eed7c0199cb44e9aad48f3ad4a31df41, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732340497173 2024-11-23T05:41:40,765 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c3ce8d689ec440cabceb6e94d4ec7540, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732340497173 2024-11-23T05:41:40,765 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0cabf826ea0041c9ac656f88b8204d73, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732340497207 2024-11-23T05:41:40,765 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 09e77500c9604211b89ef0af56a3cabb, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732340497207 2024-11-23T05:41:40,765 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9f81c4f8f79b436f9ca8e0dfc5721174, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340499383 2024-11-23T05:41:40,766 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4b964a1f09a74979abce8e822152e3e8, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340499383 2024-11-23T05:41:40,771 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:40,773 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#B#compaction#296 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:40,773 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/c4cfba76357a46b18171dc2539f18c40 is 50, key is test_row_0/B:col10/1732340499383/Put/seqid=0 2024-11-23T05:41:40,773 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123b01b8dd6731a422db33cab21d5eefe10_a9d4eac7c30b3b93cecf8ac4d22b4ff7 store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:40,775 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123b01b8dd6731a422db33cab21d5eefe10_a9d4eac7c30b3b93cecf8ac4d22b4ff7, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:40,775 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b01b8dd6731a422db33cab21d5eefe10_a9d4eac7c30b3b93cecf8ac4d22b4ff7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:40,784 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742174_1350 (size=12104) 2024-11-23T05:41:40,789 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742175_1351 (size=4469) 2024-11-23T05:41:40,792 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/c4cfba76357a46b18171dc2539f18c40 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/c4cfba76357a46b18171dc2539f18c40 2024-11-23T05:41:40,796 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/B of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into c4cfba76357a46b18171dc2539f18c40(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:40,796 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:40,796 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, priority=13, startTime=1732340500763; duration=0sec 2024-11-23T05:41:40,796 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:40,796 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:B 2024-11-23T05:41:40,796 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:40,796 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 33659 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:40,796 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/C is initiating minor compaction (all files) 2024-11-23T05:41:40,796 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/C in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:40,797 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3b400278b0444cf98406c79d80907ffc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1b7d4d104e9e4dd8a787df4319ae87c7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ee7ca6edbba24a788e18b20c8bc5e663] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=32.9 K 2024-11-23T05:41:40,797 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b400278b0444cf98406c79d80907ffc, keycount=100, bloomtype=ROW, size=9.4 K, encoding=NONE, compression=NONE, seqNum=15, earliestPutTs=1732340497173 2024-11-23T05:41:40,797 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1b7d4d104e9e4dd8a787df4319ae87c7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732340497207 2024-11-23T05:41:40,797 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ee7ca6edbba24a788e18b20c8bc5e663, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340499383 2024-11-23T05:41:40,806 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#C#compaction#298 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:40,807 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/ca007131bb79469ba38e29827efeddca is 50, key is test_row_0/C:col10/1732340499383/Put/seqid=0 2024-11-23T05:41:40,812 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742176_1352 (size=12104) 2024-11-23T05:41:41,188 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#A#compaction#297 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:41,189 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/2bc1432dd0e14281a549510acb09ae49 is 175, key is test_row_0/A:col10/1732340499383/Put/seqid=0 2024-11-23T05:41:41,192 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742177_1353 (size=31058) 2024-11-23T05:41:41,216 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/ca007131bb79469ba38e29827efeddca as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ca007131bb79469ba38e29827efeddca 2024-11-23T05:41:41,219 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/C of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into ca007131bb79469ba38e29827efeddca(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:41,219 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:41,219 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, priority=13, startTime=1732340500764; duration=0sec 2024-11-23T05:41:41,219 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:41,219 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:C 2024-11-23T05:41:41,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=99 2024-11-23T05:41:41,336 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:41,337 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:41,338 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees 2024-11-23T05:41:41,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T05:41:41,339 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:41,340 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=101, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:41,340 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=102, ppid=101, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:41,420 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:41,420 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:41:41,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:41,427 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ea0e1d592e9548c58e6b42d26f257e51_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340499525/Put/seqid=0 2024-11-23T05:41:41,434 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742178_1354 (size=14594) 2024-11-23T05:41:41,435 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:41,441 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123ea0e1d592e9548c58e6b42d26f257e51_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ea0e1d592e9548c58e6b42d26f257e51_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:41,442 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/2eb4412325ac416a91e71e51924d5797, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:41,443 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/2eb4412325ac416a91e71e51924d5797 is 175, key is test_row_0/A:col10/1732340499525/Put/seqid=0 2024-11-23T05:41:41,446 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742179_1355 (size=39549) 2024-11-23T05:41:41,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T05:41:41,448 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=79, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/2eb4412325ac416a91e71e51924d5797 2024-11-23T05:41:41,454 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/1a5831a6ad684128bdb6aae39d7deefa is 50, key is test_row_0/B:col10/1732340499525/Put/seqid=0 2024-11-23T05:41:41,458 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742180_1356 (size=12001) 2024-11-23T05:41:41,476 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:41,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340561472, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,477 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:41,492 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T05:41:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,492 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,492 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:41,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340561586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,592 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:41,596 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/2bc1432dd0e14281a549510acb09ae49 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2bc1432dd0e14281a549510acb09ae49 2024-11-23T05:41:41,599 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/A of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 2bc1432dd0e14281a549510acb09ae49(size=30.3 K), total size for store is 30.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:41,599 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:41,599 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, priority=13, startTime=1732340500763; duration=0sec 2024-11-23T05:41:41,599 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:41,599 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:A 2024-11-23T05:41:41,644 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T05:41:41,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:41,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,645 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,645 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T05:41:41,719 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340561716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:41,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340561716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:41,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:41,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340561716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:41,720 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:41,720 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,720 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,720 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,721 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:41,731 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:41,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340561726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:41,731 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,732 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:41,797 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T05:41:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:41,797 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,797 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,798 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:41,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340561796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:41,803 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:41,859 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/1a5831a6ad684128bdb6aae39d7deefa 2024-11-23T05:41:41,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/8421f729344a4f07940f3ce683f904b7 is 50, key is test_row_0/C:col10/1732340499525/Put/seqid=0 2024-11-23T05:41:41,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742181_1357 (size=12001) 2024-11-23T05:41:41,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T05:41:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:41,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:41,950 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:41,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T05:41:42,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T05:41:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:42,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:42,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:42,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:42,120 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:42,120 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340562116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:42,120 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:42,256 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T05:41:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:42,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] handler.RSProcedureHandler(58): pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:42,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=102 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:42,257 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=102 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:42,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=79 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/8421f729344a4f07940f3ce683f904b7 2024-11-23T05:41:42,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/2eb4412325ac416a91e71e51924d5797 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2eb4412325ac416a91e71e51924d5797 2024-11-23T05:41:42,282 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2eb4412325ac416a91e71e51924d5797, entries=200, sequenceid=79, filesize=38.6 K 2024-11-23T05:41:42,283 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/1a5831a6ad684128bdb6aae39d7deefa as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1a5831a6ad684128bdb6aae39d7deefa 2024-11-23T05:41:42,287 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1a5831a6ad684128bdb6aae39d7deefa, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T05:41:42,288 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/8421f729344a4f07940f3ce683f904b7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/8421f729344a4f07940f3ce683f904b7 2024-11-23T05:41:42,294 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/8421f729344a4f07940f3ce683f904b7, entries=150, sequenceid=79, filesize=11.7 K 2024-11-23T05:41:42,295 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 875ms, sequenceid=79, compaction requested=false 2024-11-23T05:41:42,295 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:42,409 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=102 2024-11-23T05:41:42,409 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:42,409 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:41:42,418 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fe5ec454a9ce4242b10971b83e7ea73a_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340501471/Put/seqid=0 2024-11-23T05:41:42,422 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742182_1358 (size=12154) 2024-11-23T05:41:42,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T05:41:42,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:42,630 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:42,767 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:42,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340562756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:42,768 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:42,823 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:42,826 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123fe5ec454a9ce4242b10971b83e7ea73a_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fe5ec454a9ce4242b10971b83e7ea73a_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8d0e368e4b3541e09a1804ab8e2806e1, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:42,827 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8d0e368e4b3541e09a1804ab8e2806e1 is 175, key is test_row_0/A:col10/1732340501471/Put/seqid=0 2024-11-23T05:41:42,830 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742183_1359 (size=30955) 2024-11-23T05:41:42,883 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:42,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340562876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:42,884 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:43,100 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:43,100 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340563096, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,101 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:43,231 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=91, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8d0e368e4b3541e09a1804ab8e2806e1 2024-11-23T05:41:43,237 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/4e923494fe2d42fb9a4f9e537582055b is 50, key is test_row_0/B:col10/1732340501471/Put/seqid=0 2024-11-23T05:41:43,240 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742184_1360 (size=12001) 2024-11-23T05:41:43,409 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:43,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340563406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,410 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:43,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T05:41:43,640 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/4e923494fe2d42fb9a4f9e537582055b 2024-11-23T05:41:43,647 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/74083749b70f4b8187fdcc67d95f4d37 is 50, key is test_row_0/C:col10/1732340501471/Put/seqid=0 2024-11-23T05:41:43,654 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742185_1361 (size=12001) 2024-11-23T05:41:43,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T05:41:43,697 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T05:41:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340563736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:43,743 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:43,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340563736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:43,743 WARN [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:43,744 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,744 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:43,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340563745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:43,754 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:43,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340563746, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:43,754 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4205 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,755 WARN [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4204 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,755 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:43,921 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:43,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340563916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:43,921 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:44,055 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=91 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/74083749b70f4b8187fdcc67d95f4d37 2024-11-23T05:41:44,059 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8d0e368e4b3541e09a1804ab8e2806e1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8d0e368e4b3541e09a1804ab8e2806e1 2024-11-23T05:41:44,062 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8d0e368e4b3541e09a1804ab8e2806e1, entries=150, sequenceid=91, filesize=30.2 K 2024-11-23T05:41:44,063 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/4e923494fe2d42fb9a4f9e537582055b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4e923494fe2d42fb9a4f9e537582055b 2024-11-23T05:41:44,066 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4e923494fe2d42fb9a4f9e537582055b, entries=150, sequenceid=91, filesize=11.7 K 2024-11-23T05:41:44,067 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/74083749b70f4b8187fdcc67d95f4d37 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/74083749b70f4b8187fdcc67d95f4d37 2024-11-23T05:41:44,072 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/74083749b70f4b8187fdcc67d95f4d37, entries=150, sequenceid=91, filesize=11.7 K 2024-11-23T05:41:44,073 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1664ms, sequenceid=91, compaction requested=true 2024-11-23T05:41:44,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:44,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:44,073 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=102}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=102 2024-11-23T05:41:44,073 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=102 2024-11-23T05:41:44,075 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=102, resume processing ppid=101 2024-11-23T05:41:44,076 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=102, ppid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.7340 sec 2024-11-23T05:41:44,077 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=101, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=101, table=TestAcidGuarantees in 2.7390 sec 2024-11-23T05:41:44,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:44,942 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:44,942 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:41:44,949 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112306e7a4e0de624408a99f09da2a77d482_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340502751/Put/seqid=0 2024-11-23T05:41:44,952 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742186_1362 (size=14594) 2024-11-23T05:41:44,952 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:44,956 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112306e7a4e0de624408a99f09da2a77d482_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112306e7a4e0de624408a99f09da2a77d482_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:44,957 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/100506c4cf2b4ff383cc5832bf071d21, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:44,957 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/100506c4cf2b4ff383cc5832bf071d21 is 175, key is test_row_0/A:col10/1732340502751/Put/seqid=0 2024-11-23T05:41:44,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742187_1363 (size=39549) 2024-11-23T05:41:44,960 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=117, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/100506c4cf2b4ff383cc5832bf071d21 2024-11-23T05:41:44,967 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/ee9e4c34db17499ea0d39a70ee6e4ba1 is 50, key is test_row_0/B:col10/1732340502751/Put/seqid=0 2024-11-23T05:41:44,972 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742188_1364 (size=12001) 2024-11-23T05:41:45,001 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:45,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340564998, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,002 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:45,112 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:45,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340565106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,112 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:45,323 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:45,324 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340565315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,324 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:45,373 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/ee9e4c34db17499ea0d39a70ee6e4ba1 2024-11-23T05:41:45,379 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/6c7d9ff858404c92827b678f5a6ab28c is 50, key is test_row_0/C:col10/1732340502751/Put/seqid=0 2024-11-23T05:41:45,382 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742189_1365 (size=12001) 2024-11-23T05:41:45,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=101 2024-11-23T05:41:45,506 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:45,507 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:45,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees 2024-11-23T05:41:45,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T05:41:45,508 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:45,509 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=103, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:45,509 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=104, ppid=103, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:45,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T05:41:45,638 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:45,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340565636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:45,639 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:45,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T05:41:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:45,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] handler.RSProcedureHandler(58): pid=104 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:45,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=104 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:45,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=104 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:45,783 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=117 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/6c7d9ff858404c92827b678f5a6ab28c 2024-11-23T05:41:45,786 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/100506c4cf2b4ff383cc5832bf071d21 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/100506c4cf2b4ff383cc5832bf071d21 2024-11-23T05:41:45,789 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/100506c4cf2b4ff383cc5832bf071d21, entries=200, sequenceid=117, filesize=38.6 K 2024-11-23T05:41:45,790 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/ee9e4c34db17499ea0d39a70ee6e4ba1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/ee9e4c34db17499ea0d39a70ee6e4ba1 2024-11-23T05:41:45,793 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/ee9e4c34db17499ea0d39a70ee6e4ba1, entries=150, sequenceid=117, filesize=11.7 K 2024-11-23T05:41:45,793 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/6c7d9ff858404c92827b678f5a6ab28c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/6c7d9ff858404c92827b678f5a6ab28c 2024-11-23T05:41:45,799 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/6c7d9ff858404c92827b678f5a6ab28c, entries=150, sequenceid=117, filesize=11.7 K 2024-11-23T05:41:45,799 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 857ms, sequenceid=117, compaction requested=true 2024-11-23T05:41:45,799 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:45,800 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:45,800 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:45,800 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:45,801 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141111 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:45,801 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:45,801 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/B is initiating minor compaction (all files) 2024-11-23T05:41:45,801 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/A is initiating minor compaction (all files) 2024-11-23T05:41:45,801 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/A in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:45,801 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/B in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:45,801 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2bc1432dd0e14281a549510acb09ae49, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2eb4412325ac416a91e71e51924d5797, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8d0e368e4b3541e09a1804ab8e2806e1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/100506c4cf2b4ff383cc5832bf071d21] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=137.8 K 2024-11-23T05:41:45,801 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/c4cfba76357a46b18171dc2539f18c40, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1a5831a6ad684128bdb6aae39d7deefa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4e923494fe2d42fb9a4f9e537582055b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/ee9e4c34db17499ea0d39a70ee6e4ba1] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=47.0 K 2024-11-23T05:41:45,801 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:45,801 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2bc1432dd0e14281a549510acb09ae49, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2eb4412325ac416a91e71e51924d5797, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8d0e368e4b3541e09a1804ab8e2806e1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/100506c4cf2b4ff383cc5832bf071d21] 2024-11-23T05:41:45,801 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting c4cfba76357a46b18171dc2539f18c40, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340499383 2024-11-23T05:41:45,801 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bc1432dd0e14281a549510acb09ae49, keycount=150, bloomtype=ROW, size=30.3 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340499383 2024-11-23T05:41:45,802 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a5831a6ad684128bdb6aae39d7deefa, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732340499525 2024-11-23T05:41:45,802 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2eb4412325ac416a91e71e51924d5797, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732340499525 2024-11-23T05:41:45,802 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4e923494fe2d42fb9a4f9e537582055b, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732340501461 2024-11-23T05:41:45,802 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8d0e368e4b3541e09a1804ab8e2806e1, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732340501461 2024-11-23T05:41:45,803 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ee9e4c34db17499ea0d39a70ee6e4ba1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732340502751 2024-11-23T05:41:45,803 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 100506c4cf2b4ff383cc5832bf071d21, keycount=200, bloomtype=ROW, size=38.6 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732340502744 2024-11-23T05:41:45,809 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:45,811 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#B#compaction#308 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:45,812 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/2b404e3281654d828b76cf8e6c26f184 is 50, key is test_row_0/B:col10/1732340502751/Put/seqid=0 2024-11-23T05:41:45,812 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411230f169bc777684cb4845263e7f7287686_a9d4eac7c30b3b93cecf8ac4d22b4ff7 store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:45,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=104 2024-11-23T05:41:45,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:45,814 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T05:41:45,814 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411230f169bc777684cb4845263e7f7287686_a9d4eac7c30b3b93cecf8ac4d22b4ff7, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:45,815 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230f169bc777684cb4845263e7f7287686_a9d4eac7c30b3b93cecf8ac4d22b4ff7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:45,824 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742190_1366 (size=12241) 2024-11-23T05:41:45,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T05:41:45,836 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742191_1367 (size=4469) 2024-11-23T05:41:45,838 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#A#compaction#309 average throughput is 0.84 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:45,838 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/6f48876899be422a91a70db0f19cb7a0 is 175, key is test_row_0/A:col10/1732340502751/Put/seqid=0 2024-11-23T05:41:45,844 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123aff2aeedde4f47a3981834c1ffa80221_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340504971/Put/seqid=0 2024-11-23T05:41:45,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742192_1368 (size=31195) 2024-11-23T05:41:45,850 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/6f48876899be422a91a70db0f19cb7a0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/6f48876899be422a91a70db0f19cb7a0 2024-11-23T05:41:45,854 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742193_1369 (size=12154) 2024-11-23T05:41:45,856 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/A of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 6f48876899be422a91a70db0f19cb7a0(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:45,856 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:45,856 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, priority=12, startTime=1732340505799; duration=0sec 2024-11-23T05:41:45,856 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:45,856 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:A 2024-11-23T05:41:45,856 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:45,857 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48107 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:45,857 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/C is initiating minor compaction (all files) 2024-11-23T05:41:45,857 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/C in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:45,857 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ca007131bb79469ba38e29827efeddca, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/8421f729344a4f07940f3ce683f904b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/74083749b70f4b8187fdcc67d95f4d37, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/6c7d9ff858404c92827b678f5a6ab28c] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=47.0 K 2024-11-23T05:41:45,857 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ca007131bb79469ba38e29827efeddca, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340499383 2024-11-23T05:41:45,858 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8421f729344a4f07940f3ce683f904b7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=79, earliestPutTs=1732340499525 2024-11-23T05:41:45,858 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74083749b70f4b8187fdcc67d95f4d37, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=91, earliestPutTs=1732340501461 2024-11-23T05:41:45,858 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6c7d9ff858404c92827b678f5a6ab28c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732340502751 2024-11-23T05:41:45,865 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#C#compaction#311 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:45,866 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/1ccd2eccb2194b28aea511d0062bdb75 is 50, key is test_row_0/C:col10/1732340502751/Put/seqid=0 2024-11-23T05:41:45,869 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742194_1370 (size=12241) 2024-11-23T05:41:46,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T05:41:46,149 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:46,149 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:46,228 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/2b404e3281654d828b76cf8e6c26f184 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2b404e3281654d828b76cf8e6c26f184 2024-11-23T05:41:46,232 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/B of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 2b404e3281654d828b76cf8e6c26f184(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:46,232 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:46,232 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, priority=12, startTime=1732340505800; duration=0sec 2024-11-23T05:41:46,232 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:46,232 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:B 2024-11-23T05:41:46,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:46,258 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123aff2aeedde4f47a3981834c1ffa80221_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123aff2aeedde4f47a3981834c1ffa80221_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:46,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/55329dd72dae4aeea09ac5f2aed95888, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:46,259 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/55329dd72dae4aeea09ac5f2aed95888 is 175, key is test_row_0/A:col10/1732340504971/Put/seqid=0 2024-11-23T05:41:46,262 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742195_1371 (size=30955) 2024-11-23T05:41:46,272 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/1ccd2eccb2194b28aea511d0062bdb75 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1ccd2eccb2194b28aea511d0062bdb75 2024-11-23T05:41:46,275 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/C of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 1ccd2eccb2194b28aea511d0062bdb75(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:46,275 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:46,275 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, priority=12, startTime=1732340505800; duration=0sec 2024-11-23T05:41:46,275 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:46,276 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:C 2024-11-23T05:41:46,277 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:46,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340566268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,277 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:46,393 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:46,393 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340566386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,394 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:46,610 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:46,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340566606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:46,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,611 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:46,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T05:41:46,662 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=127, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/55329dd72dae4aeea09ac5f2aed95888 2024-11-23T05:41:46,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/1dfd00bce5d748e591a9be819129ec87 is 50, key is test_row_0/B:col10/1732340504971/Put/seqid=0 2024-11-23T05:41:46,670 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742196_1372 (size=12001) 2024-11-23T05:41:46,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:46,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340566916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:46,922 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:46,923 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,071 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/1dfd00bce5d748e591a9be819129ec87 2024-11-23T05:41:47,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/b49a963d80db4835877ef63bebf5adf5 is 50, key is test_row_0/C:col10/1732340504971/Put/seqid=0 2024-11-23T05:41:47,136 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742197_1373 (size=12001) 2024-11-23T05:41:47,441 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340567435, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,441 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,536 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=127 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/b49a963d80db4835877ef63bebf5adf5 2024-11-23T05:41:47,540 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/55329dd72dae4aeea09ac5f2aed95888 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/55329dd72dae4aeea09ac5f2aed95888 2024-11-23T05:41:47,542 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/55329dd72dae4aeea09ac5f2aed95888, entries=150, sequenceid=127, filesize=30.2 K 2024-11-23T05:41:47,543 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/1dfd00bce5d748e591a9be819129ec87 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1dfd00bce5d748e591a9be819129ec87 2024-11-23T05:41:47,546 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1dfd00bce5d748e591a9be819129ec87, entries=150, sequenceid=127, filesize=11.7 K 2024-11-23T05:41:47,547 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/b49a963d80db4835877ef63bebf5adf5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/b49a963d80db4835877ef63bebf5adf5 2024-11-23T05:41:47,550 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/b49a963d80db4835877ef63bebf5adf5, entries=150, sequenceid=127, filesize=11.7 K 2024-11-23T05:41:47,550 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1736ms, sequenceid=127, compaction requested=false 2024-11-23T05:41:47,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:47,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:47,551 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=104}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=104 2024-11-23T05:41:47,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=104 2024-11-23T05:41:47,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=104, resume processing ppid=103 2024-11-23T05:41:47,553 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=104, ppid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0430 sec 2024-11-23T05:41:47,554 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=103, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=103, table=TestAcidGuarantees in 2.0460 sec 2024-11-23T05:41:47,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=103 2024-11-23T05:41:47,656 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:47,657 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:47,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees 2024-11-23T05:41:47,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T05:41:47,658 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:47,659 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=105, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:47,659 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=106, ppid=105, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:47,758 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:47,759 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:47,759 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T05:41:47,766 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dbb992df02504a60a10c4bf6a04c8a03_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340506267/Put/seqid=0 2024-11-23T05:41:47,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T05:41:47,774 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742198_1374 (size=14794) 2024-11-23T05:41:47,785 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340567776, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41692 deadline: 1732340567777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340567777, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,786 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,786 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 8, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 8243 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340567781, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,786 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,787 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:47,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:47,811 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:47,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:47,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:47,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340567896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340567896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,902 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,902 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340567896, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:47,903 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:47,963 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:47,963 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:47,963 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:47,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:47,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T05:41:48,111 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340568106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340568106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,112 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340568106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,112 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,116 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:48,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:48,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,116 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,174 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:48,177 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123dbb992df02504a60a10c4bf6a04c8a03_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dbb992df02504a60a10c4bf6a04c8a03_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:48,177 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/a198d0b2e32046448f4c270336d08a84, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:48,178 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/a198d0b2e32046448f4c270336d08a84 is 175, key is test_row_0/A:col10/1732340506267/Put/seqid=0 2024-11-23T05:41:48,181 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742199_1375 (size=39749) 2024-11-23T05:41:48,268 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,268 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,268 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,269 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T05:41:48,421 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:48,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:48,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,421 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,421 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340568416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340568416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,422 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340568416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,422 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,449 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340568446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,449 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2181 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,450 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,573 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:48,573 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,573 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,574 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,582 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=157, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/a198d0b2e32046448f4c270336d08a84 2024-11-23T05:41:48,588 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/874bcf5d914a4908ae939e087b515bcf is 50, key is test_row_0/B:col10/1732340506267/Put/seqid=0 2024-11-23T05:41:48,591 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742200_1376 (size=12151) 2024-11-23T05:41:48,726 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,726 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,726 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T05:41:48,878 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:48,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:48,878 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,879 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,879 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:48,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340568925, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340568936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,941 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340568936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:48,941 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:48,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/874bcf5d914a4908ae939e087b515bcf 2024-11-23T05:41:48,998 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/c6b15529b2fb49c49277c1d234bb5550 is 50, key is test_row_0/C:col10/1732340506267/Put/seqid=0 2024-11-23T05:41:49,003 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742201_1377 (size=12151) 2024-11-23T05:41:49,031 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,031 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,031 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,184 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,184 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,184 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,185 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,336 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,337 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] handler.RSProcedureHandler(58): pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,337 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=106 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,337 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=106 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:49,404 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=157 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/c6b15529b2fb49c49277c1d234bb5550 2024-11-23T05:41:49,408 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/a198d0b2e32046448f4c270336d08a84 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/a198d0b2e32046448f4c270336d08a84 2024-11-23T05:41:49,414 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/a198d0b2e32046448f4c270336d08a84, entries=200, sequenceid=157, filesize=38.8 K 2024-11-23T05:41:49,415 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/874bcf5d914a4908ae939e087b515bcf as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/874bcf5d914a4908ae939e087b515bcf 2024-11-23T05:41:49,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,418 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/874bcf5d914a4908ae939e087b515bcf, entries=150, sequenceid=157, filesize=11.9 K 2024-11-23T05:41:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,418 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/c6b15529b2fb49c49277c1d234bb5550 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c6b15529b2fb49c49277c1d234bb5550 2024-11-23T05:41:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,419 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,420 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,421 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c6b15529b2fb49c49277c1d234bb5550, entries=150, sequenceid=157, filesize=11.9 K 2024-11-23T05:41:49,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,422 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1663ms, sequenceid=157, compaction requested=true 2024-11-23T05:41:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,422 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:49,422 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:49,422 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:49,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:49,423 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:49,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:49,423 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,423 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 101899 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:49,423 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,423 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:49,423 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/A is initiating minor compaction (all files) 2024-11-23T05:41:49,423 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/B is initiating minor compaction (all files) 2024-11-23T05:41:49,423 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/B in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,423 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/A in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,424 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2b404e3281654d828b76cf8e6c26f184, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1dfd00bce5d748e591a9be819129ec87, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/874bcf5d914a4908ae939e087b515bcf] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=35.5 K 2024-11-23T05:41:49,424 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/6f48876899be422a91a70db0f19cb7a0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/55329dd72dae4aeea09ac5f2aed95888, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/a198d0b2e32046448f4c270336d08a84] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=99.5 K 2024-11-23T05:41:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,424 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,424 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/6f48876899be422a91a70db0f19cb7a0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/55329dd72dae4aeea09ac5f2aed95888, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/a198d0b2e32046448f4c270336d08a84] 2024-11-23T05:41:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,424 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2b404e3281654d828b76cf8e6c26f184, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732340502751 2024-11-23T05:41:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,424 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6f48876899be422a91a70db0f19cb7a0, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732340502751 2024-11-23T05:41:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,424 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1dfd00bce5d748e591a9be819129ec87, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732340504971 2024-11-23T05:41:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,424 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 55329dd72dae4aeea09ac5f2aed95888, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732340504971 2024-11-23T05:41:49,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,424 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 874bcf5d914a4908ae939e087b515bcf, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340506256 2024-11-23T05:41:49,425 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a198d0b2e32046448f4c270336d08a84, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340506256 2024-11-23T05:41:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,425 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,429 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#B#compaction#317 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,430 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,431 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/2d13d5ee5c1f4b8a9b07d85714266b3d is 50, key is test_row_0/B:col10/1732340506267/Put/seqid=0 2024-11-23T05:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,431 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,432 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123b737bf53c6e04a51bb805eecbf0ea453_a9d4eac7c30b3b93cecf8ac4d22b4ff7 store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,433 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,434 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,435 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123b737bf53c6e04a51bb805eecbf0ea453_a9d4eac7c30b3b93cecf8ac4d22b4ff7, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:49,435 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b737bf53c6e04a51bb805eecbf0ea453_a9d4eac7c30b3b93cecf8ac4d22b4ff7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742202_1378 (size=12493) 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,438 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,439 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,442 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/2d13d5ee5c1f4b8a9b07d85714266b3d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2d13d5ee5c1f4b8a9b07d85714266b3d 2024-11-23T05:41:49,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,443 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,445 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742203_1379 (size=4469) 2024-11-23T05:41:49,446 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/B of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 2d13d5ee5c1f4b8a9b07d85714266b3d(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:49,446 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:49,446 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, priority=13, startTime=1732340509422; duration=0sec 2024-11-23T05:41:49,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,446 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:49,446 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:B 2024-11-23T05:41:49,446 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:49,447 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36393 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:49,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,447 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/C is initiating minor compaction (all files) 2024-11-23T05:41:49,447 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/C in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,447 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1ccd2eccb2194b28aea511d0062bdb75, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/b49a963d80db4835877ef63bebf5adf5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c6b15529b2fb49c49277c1d234bb5550] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=35.5 K 2024-11-23T05:41:49,448 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1ccd2eccb2194b28aea511d0062bdb75, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=117, earliestPutTs=1732340502751 2024-11-23T05:41:49,448 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b49a963d80db4835877ef63bebf5adf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=127, earliestPutTs=1732340504971 2024-11-23T05:41:49,448 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting c6b15529b2fb49c49277c1d234bb5550, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340506256 2024-11-23T05:41:49,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,449 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,453 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#C#compaction#319 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/2ea81b50fd1e460bacbdb33768086661 is 50, key is test_row_0/C:col10/1732340506267/Put/seqid=0 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,455 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742204_1380 (size=12493) 2024-11-23T05:41:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,460 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,461 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,463 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,464 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,465 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,467 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,468 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,470 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,474 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,476 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,477 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,478 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,479 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,483 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,484 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,487 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,489 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=106 2024-11-23T05:41:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,489 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,490 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-23T05:41:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,491 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,492 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,493 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,494 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,495 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,505 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,507 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236bfc4389b057424792f0ac981af6fc68_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_1/A:col10/1732340507780/Put/seqid=0 2024-11-23T05:41:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,510 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742205_1381 (size=9814) 2024-11-23T05:41:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,510 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,512 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,514 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236bfc4389b057424792f0ac981af6fc68_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236bfc4389b057424792f0ac981af6fc68_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,514 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/7a3684fbbc7f4a7db477a3b9e248ade8, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:49,514 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,515 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/7a3684fbbc7f4a7db477a3b9e248ade8 is 175, key is test_row_1/A:col10/1732340507780/Put/seqid=0 2024-11-23T05:41:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,515 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,517 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,518 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742206_1382 (size=22461) 2024-11-23T05:41:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,519 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,520 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,521 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,522 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,523 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,524 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,526 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,529 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,530 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,531 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,537 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,541 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,542 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,546 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,547 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,550 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,551 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,552 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,553 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,554 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,555 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,556 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,558 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,563 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,564 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,565 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,567 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,569 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,570 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,571 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,572 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,573 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,574 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,575 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,579 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,580 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,581 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,583 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,584 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,585 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,590 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,591 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,595 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,597 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,600 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,601 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,611 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,612 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,614 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,616 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,622 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,624 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,626 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,629 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,630 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,631 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,632 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,633 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,634 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,635 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,643 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,644 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,645 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,652 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,653 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,654 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,656 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,658 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,660 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,663 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,664 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,665 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,666 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,672 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,673 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,681 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,682 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,683 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,684 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,685 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,686 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,687 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,688 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,692 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,693 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,694 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,697 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,700 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,701 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,702 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,703 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,704 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,705 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,707 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,708 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,709 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,716 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,721 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,724 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,731 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,732 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,734 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,737 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,738 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,739 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,743 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,744 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,747 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,748 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,749 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,751 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,752 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,753 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,754 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,755 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,759 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,760 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,762 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,768 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,769 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,770 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,772 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,773 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,774 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,775 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,777 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,778 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,779 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,781 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,783 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,784 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,785 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,787 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,790 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,791 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,792 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,793 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,794 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,795 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,800 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,801 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,805 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T05:41:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,806 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,807 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,813 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,817 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,820 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,822 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,823 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,825 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,829 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,833 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,835 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,836 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,837 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,840 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,846 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#A#compaction#318 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:49,847 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ebe7cad7685f450a9730bcfd88a69001 is 175, key is test_row_0/A:col10/1732340506267/Put/seqid=0 2024-11-23T05:41:49,847 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,849 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,850 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,851 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,852 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,853 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,854 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742207_1383 (size=31447) 2024-11-23T05:41:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,855 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,856 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,857 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,860 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ebe7cad7685f450a9730bcfd88a69001 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ebe7cad7685f450a9730bcfd88a69001 2024-11-23T05:41:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,861 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,862 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,864 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,865 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,866 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/A of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into ebe7cad7685f450a9730bcfd88a69001(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:49,866 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:49,866 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, priority=13, startTime=1732340509422; duration=0sec 2024-11-23T05:41:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,866 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:49,866 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:A 2024-11-23T05:41:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,866 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,867 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/2ea81b50fd1e460bacbdb33768086661 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/2ea81b50fd1e460bacbdb33768086661 2024-11-23T05:41:49,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,868 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,871 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,872 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/C of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 2ea81b50fd1e460bacbdb33768086661(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:49,872 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,872 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, priority=13, startTime=1732340509423; duration=0sec 2024-11-23T05:41:49,872 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:49,872 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:C 2024-11-23T05:41:49,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,874 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,875 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,883 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,884 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,889 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,890 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,891 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,892 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,893 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,895 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,898 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,900 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,901 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,902 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,903 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,904 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,905 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,908 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,909 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,910 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,911 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,912 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,913 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,914 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,919 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,920 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=167, memsize=13.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/7a3684fbbc7f4a7db477a3b9e248ade8 2024-11-23T05:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,920 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,923 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,926 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/7d432e597ef44bebb7711f4a390f99be is 50, key is test_row_1/B:col10/1732340507780/Put/seqid=0 2024-11-23T05:41:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,927 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,929 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,930 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,937 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,938 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,939 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,940 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742208_1384 (size=9757) 2024-11-23T05:41:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,940 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/7d432e597ef44bebb7711f4a390f99be 2024-11-23T05:41:49,940 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,941 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,942 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,943 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,944 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,945 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,948 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,949 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/3aebc5abe39a4e2987b9f96eef7b325d is 50, key is test_row_1/C:col10/1732340507780/Put/seqid=0 2024-11-23T05:41:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,953 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,954 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,955 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,957 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,958 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,959 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742209_1385 (size=9757) 2024-11-23T05:41:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,960 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,961 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=167 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/3aebc5abe39a4e2987b9f96eef7b325d 2024-11-23T05:41:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,961 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,962 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,964 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,965 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/7a3684fbbc7f4a7db477a3b9e248ade8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/7a3684fbbc7f4a7db477a3b9e248ade8 2024-11-23T05:41:49,965 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,969 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/7a3684fbbc7f4a7db477a3b9e248ade8, entries=100, sequenceid=167, filesize=21.9 K 2024-11-23T05:41:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,969 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,970 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/7d432e597ef44bebb7711f4a390f99be as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7d432e597ef44bebb7711f4a390f99be 2024-11-23T05:41:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,970 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,971 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,972 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,973 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,974 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7d432e597ef44bebb7711f4a390f99be, entries=100, sequenceid=167, filesize=9.5 K 2024-11-23T05:41:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,974 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/3aebc5abe39a4e2987b9f96eef7b325d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3aebc5abe39a4e2987b9f96eef7b325d 2024-11-23T05:41:49,975 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3aebc5abe39a4e2987b9f96eef7b325d, entries=100, sequenceid=167, filesize=9.5 K 2024-11-23T05:41:49,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=6.71 KB/6870 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 490ms, sequenceid=167, compaction requested=false 2024-11-23T05:41:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:49,981 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=106}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=106 2024-11-23T05:41:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=106 2024-11-23T05:41:49,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=106, resume processing ppid=105 2024-11-23T05:41:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,984 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=106, ppid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3230 sec 2024-11-23T05:41:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,986 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=105, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=105, table=TestAcidGuarantees in 2.3280 sec 2024-11-23T05:41:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,987 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,988 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,990 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,991 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,992 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,993 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,994 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,995 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,997 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:49,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,000 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,001 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,002 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,003 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,004 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,005 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,006 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,008 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,012 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,013 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,014 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,015 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,016 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,017 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,022 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,023 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,024 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,025 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,027 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:50,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,034 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:50,034 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:41:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,035 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,036 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,043 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a9f1da1d9200411299446153a0ba2497_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340510033/Put/seqid=0 2024-11-23T05:41:50,074 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742211_1387 (size=24758) 2024-11-23T05:41:50,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340570101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,113 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,113 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340570103, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,113 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,115 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340570104, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,115 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340570216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,222 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340570216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,223 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,233 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340570225, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,233 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,234 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340570426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340570426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,429 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,430 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340570445, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,451 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,469 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,469 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340570466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,470 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in a9d4eac7c30b3b93cecf8ac4d22b4ff7 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4201 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:41:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,470 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,474 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:50,477 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123a9f1da1d9200411299446153a0ba2497_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a9f1da1d9200411299446153a0ba2497_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:50,478 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ae1ae9397cf44ce6be8e89f26808675c, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:50,478 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ae1ae9397cf44ce6be8e89f26808675c is 175, key is test_row_0/A:col10/1732340510033/Put/seqid=0 2024-11-23T05:41:50,480 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742210_1386 (size=74395) 2024-11-23T05:41:50,480 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=180, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ae1ae9397cf44ce6be8e89f26808675c 2024-11-23T05:41:50,490 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/7ae28256a76241558fe737a73239ebaa is 50, key is test_row_0/B:col10/1732340510033/Put/seqid=0 2024-11-23T05:41:50,493 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742212_1388 (size=12151) 2024-11-23T05:41:50,741 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340570736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,742 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340570736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,742 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,763 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:50,763 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340570756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:50,763 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:50,894 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/7ae28256a76241558fe737a73239ebaa 2024-11-23T05:41:50,900 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/e15e8592e92741599f30547e70ee8799 is 50, key is test_row_0/C:col10/1732340510033/Put/seqid=0 2024-11-23T05:41:50,903 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742213_1389 (size=12151) 2024-11-23T05:41:51,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:51,260 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340571256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:51,260 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:51,261 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340571256, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:51,261 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:51,270 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:51,270 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340571266, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:51,270 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:51,303 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=180 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/e15e8592e92741599f30547e70ee8799 2024-11-23T05:41:51,306 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ae1ae9397cf44ce6be8e89f26808675c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ae1ae9397cf44ce6be8e89f26808675c 2024-11-23T05:41:51,309 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ae1ae9397cf44ce6be8e89f26808675c, entries=400, sequenceid=180, filesize=72.7 K 2024-11-23T05:41:51,310 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/7ae28256a76241558fe737a73239ebaa as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7ae28256a76241558fe737a73239ebaa 2024-11-23T05:41:51,312 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7ae28256a76241558fe737a73239ebaa, entries=150, sequenceid=180, filesize=11.9 K 2024-11-23T05:41:51,313 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/e15e8592e92741599f30547e70ee8799 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e15e8592e92741599f30547e70ee8799 2024-11-23T05:41:51,316 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e15e8592e92741599f30547e70ee8799, entries=150, sequenceid=180, filesize=11.9 K 2024-11-23T05:41:51,316 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1282ms, sequenceid=180, compaction requested=true 2024-11-23T05:41:51,316 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:51,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:51,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:51,316 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:51,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:51,316 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:51,316 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:51,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:51,317 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:51,317 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 128303 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:51,317 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:51,317 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/A is initiating minor compaction (all files) 2024-11-23T05:41:51,317 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/B is initiating minor compaction (all files) 2024-11-23T05:41:51,317 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/B in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:51,317 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/A in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:51,317 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2d13d5ee5c1f4b8a9b07d85714266b3d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7d432e597ef44bebb7711f4a390f99be, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7ae28256a76241558fe737a73239ebaa] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=33.6 K 2024-11-23T05:41:51,317 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ebe7cad7685f450a9730bcfd88a69001, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/7a3684fbbc7f4a7db477a3b9e248ade8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ae1ae9397cf44ce6be8e89f26808675c] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=125.3 K 2024-11-23T05:41:51,317 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:51,318 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ebe7cad7685f450a9730bcfd88a69001, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/7a3684fbbc7f4a7db477a3b9e248ade8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ae1ae9397cf44ce6be8e89f26808675c] 2024-11-23T05:41:51,318 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2d13d5ee5c1f4b8a9b07d85714266b3d, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340506256 2024-11-23T05:41:51,318 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ebe7cad7685f450a9730bcfd88a69001, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340506256 2024-11-23T05:41:51,318 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7a3684fbbc7f4a7db477a3b9e248ade8, keycount=100, bloomtype=ROW, size=21.9 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732340507775 2024-11-23T05:41:51,318 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7d432e597ef44bebb7711f4a390f99be, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732340507775 2024-11-23T05:41:51,318 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 7ae28256a76241558fe737a73239ebaa, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340510024 2024-11-23T05:41:51,318 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae1ae9397cf44ce6be8e89f26808675c, keycount=400, bloomtype=ROW, size=72.7 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340509952 2024-11-23T05:41:51,323 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:51,323 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#B#compaction#326 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:51,324 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/d8b9227a53b14c5e836b9bfe37f0dded is 50, key is test_row_0/B:col10/1732340510033/Put/seqid=0 2024-11-23T05:41:51,325 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411230eecbc049ba247a1b9fabbb257a28711_a9d4eac7c30b3b93cecf8ac4d22b4ff7 store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:51,332 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411230eecbc049ba247a1b9fabbb257a28711_a9d4eac7c30b3b93cecf8ac4d22b4ff7, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:51,332 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230eecbc049ba247a1b9fabbb257a28711_a9d4eac7c30b3b93cecf8ac4d22b4ff7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:51,333 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742214_1390 (size=12595) 2024-11-23T05:41:51,339 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742215_1391 (size=4469) 2024-11-23T05:41:51,737 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/d8b9227a53b14c5e836b9bfe37f0dded as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d8b9227a53b14c5e836b9bfe37f0dded 2024-11-23T05:41:51,740 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#A#compaction#327 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:51,741 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/B of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into d8b9227a53b14c5e836b9bfe37f0dded(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:51,741 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:51,741 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, priority=13, startTime=1732340511316; duration=0sec 2024-11-23T05:41:51,741 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:51,741 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:B 2024-11-23T05:41:51,741 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:41:51,741 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/b1045438798d4a0dba3a1a96ba38e889 is 175, key is test_row_0/A:col10/1732340510033/Put/seqid=0 2024-11-23T05:41:51,742 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 34401 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:41:51,742 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/C is initiating minor compaction (all files) 2024-11-23T05:41:51,742 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/C in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:51,742 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/2ea81b50fd1e460bacbdb33768086661, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3aebc5abe39a4e2987b9f96eef7b325d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e15e8592e92741599f30547e70ee8799] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=33.6 K 2024-11-23T05:41:51,743 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2ea81b50fd1e460bacbdb33768086661, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=157, earliestPutTs=1732340506256 2024-11-23T05:41:51,743 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aebc5abe39a4e2987b9f96eef7b325d, keycount=100, bloomtype=ROW, size=9.5 K, encoding=NONE, compression=NONE, seqNum=167, earliestPutTs=1732340507775 2024-11-23T05:41:51,743 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting e15e8592e92741599f30547e70ee8799, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340510024 2024-11-23T05:41:51,748 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742216_1392 (size=31549) 2024-11-23T05:41:51,750 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#C#compaction#328 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:51,750 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/42cb7fb3b2c44009a5e8f6440682ea1f is 50, key is test_row_0/C:col10/1732340510033/Put/seqid=0 2024-11-23T05:41:51,754 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742217_1393 (size=12595) 2024-11-23T05:41:51,757 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/42cb7fb3b2c44009a5e8f6440682ea1f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/42cb7fb3b2c44009a5e8f6440682ea1f 2024-11-23T05:41:51,761 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/C of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 42cb7fb3b2c44009a5e8f6440682ea1f(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:51,761 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:51,761 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, priority=13, startTime=1732340511316; duration=0sec 2024-11-23T05:41:51,761 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:51,761 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:C 2024-11-23T05:41:51,826 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=105 2024-11-23T05:41:51,826 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:51,827 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:51,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees 2024-11-23T05:41:51,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T05:41:51,828 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:51,829 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=107, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:51,829 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=108, ppid=107, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:51,936 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T05:41:51,980 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=108 2024-11-23T05:41:51,980 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:51,980 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:41:51,986 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123efdefb043fa54b898989f633b4efd25f_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340510092/Put/seqid=0 2024-11-23T05:41:51,989 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742218_1394 (size=12304) 2024-11-23T05:41:52,146 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T05:41:52,194 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/b1045438798d4a0dba3a1a96ba38e889 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/b1045438798d4a0dba3a1a96ba38e889 2024-11-23T05:41:52,197 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/A of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into b1045438798d4a0dba3a1a96ba38e889(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:52,197 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:52,197 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, priority=13, startTime=1732340511316; duration=0sec 2024-11-23T05:41:52,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:52,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:A 2024-11-23T05:41:52,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:52,279 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:52,293 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340572290, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,297 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340572291, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,298 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,298 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340572293, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,298 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,347 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T05:41:52,389 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:52,392 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123efdefb043fa54b898989f633b4efd25f_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123efdefb043fa54b898989f633b4efd25f_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:52,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/e12a6dd90156434dbe6d96ef75f80e13, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:52,393 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/e12a6dd90156434dbe6d96ef75f80e13 is 175, key is test_row_0/A:col10/1732340510092/Put/seqid=0 2024-11-23T05:41:52,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742219_1395 (size=31105) 2024-11-23T05:41:52,404 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340572400, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,405 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,410 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340572406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,410 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,411 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,412 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340572406, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,412 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T05:41:52,617 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340572616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340572616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,618 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340572616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,618 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,802 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=207, memsize=49.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/e12a6dd90156434dbe6d96ef75f80e13 2024-11-23T05:41:52,807 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/d89e6911074d41bba2cc16177ded2808 is 50, key is test_row_0/B:col10/1732340510092/Put/seqid=0 2024-11-23T05:41:52,810 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742220_1396 (size=12151) 2024-11-23T05:41:52,931 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,931 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340572926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340572926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,932 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,932 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340572926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:52,932 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:52,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T05:41:53,210 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/d89e6911074d41bba2cc16177ded2808 2024-11-23T05:41:53,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/4afe3ec0a46a42cd94176e07cd881a07 is 50, key is test_row_0/C:col10/1732340510092/Put/seqid=0 2024-11-23T05:41:53,221 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742221_1397 (size=12151) 2024-11-23T05:41:53,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:53,440 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340573436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:53,440 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:53,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340573436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:53,441 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:53,451 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:53,452 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340573446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:53,452 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:53,620 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=207 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/4afe3ec0a46a42cd94176e07cd881a07 2024-11-23T05:41:53,624 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/e12a6dd90156434dbe6d96ef75f80e13 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/e12a6dd90156434dbe6d96ef75f80e13 2024-11-23T05:41:53,627 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/e12a6dd90156434dbe6d96ef75f80e13, entries=150, sequenceid=207, filesize=30.4 K 2024-11-23T05:41:53,628 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/d89e6911074d41bba2cc16177ded2808 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d89e6911074d41bba2cc16177ded2808 2024-11-23T05:41:53,631 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d89e6911074d41bba2cc16177ded2808, entries=150, sequenceid=207, filesize=11.9 K 2024-11-23T05:41:53,631 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/4afe3ec0a46a42cd94176e07cd881a07 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/4afe3ec0a46a42cd94176e07cd881a07 2024-11-23T05:41:53,634 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/4afe3ec0a46a42cd94176e07cd881a07, entries=150, sequenceid=207, filesize=11.9 K 2024-11-23T05:41:53,635 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1655ms, sequenceid=207, compaction requested=false 2024-11-23T05:41:53,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:53,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:53,635 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=108}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=108 2024-11-23T05:41:53,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=108 2024-11-23T05:41:53,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=108, resume processing ppid=107 2024-11-23T05:41:53,638 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=108, ppid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8080 sec 2024-11-23T05:41:53,641 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=107, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=107, table=TestAcidGuarantees in 1.8120 sec 2024-11-23T05:41:53,976 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=107 2024-11-23T05:41:53,976 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:53,977 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees 2024-11-23T05:41:53,978 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:53,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T05:41:53,979 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=109, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:53,979 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=110, ppid=109, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:54,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T05:41:54,131 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=110 2024-11-23T05:41:54,131 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:54,131 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:41:54,137 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112356de2d9f84e44c6db7ba8fe0f58f7a56_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340512292/Put/seqid=0 2024-11-23T05:41:54,140 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742222_1398 (size=12304) 2024-11-23T05:41:54,296 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T05:41:54,450 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:54,450 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:54,541 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:54,544 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112356de2d9f84e44c6db7ba8fe0f58f7a56_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112356de2d9f84e44c6db7ba8fe0f58f7a56_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:54,544 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,544 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/411ced69e10344458dc8ffbf13431922, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:54,544 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340574532, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,545 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,545 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/411ced69e10344458dc8ffbf13431922 is 175, key is test_row_0/A:col10/1732340512292/Put/seqid=0 2024-11-23T05:41:54,545 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340574533, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,545 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,547 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,548 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340574543, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,548 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742223_1399 (size=31105) 2024-11-23T05:41:54,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,549 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,557 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,557 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340574545, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,557 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T05:41:54,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,661 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340574656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,661 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340574656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340574656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,662 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340574666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,672 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340574866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,869 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,869 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340574866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,869 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,870 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340574866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,870 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,882 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:54,882 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340574876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:54,883 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:54,949 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=219, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/411ced69e10344458dc8ffbf13431922 2024-11-23T05:41:54,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/4cbd74375ce1459584e1234a906e3578 is 50, key is test_row_0/B:col10/1732340512292/Put/seqid=0 2024-11-23T05:41:54,958 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742224_1400 (size=12151) 2024-11-23T05:41:55,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T05:41:55,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340575176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,182 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,182 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340575176, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,182 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,183 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,183 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340575177, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,183 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,191 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340575186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,192 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,360 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/4cbd74375ce1459584e1234a906e3578 2024-11-23T05:41:55,369 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/970c45ab244d4f42abf40a31412fe312 is 50, key is test_row_0/C:col10/1732340512292/Put/seqid=0 2024-11-23T05:41:55,380 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742225_1401 (size=12151) 2024-11-23T05:41:55,380 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=219 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/970c45ab244d4f42abf40a31412fe312 2024-11-23T05:41:55,383 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/411ced69e10344458dc8ffbf13431922 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/411ced69e10344458dc8ffbf13431922 2024-11-23T05:41:55,386 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/411ced69e10344458dc8ffbf13431922, entries=150, sequenceid=219, filesize=30.4 K 2024-11-23T05:41:55,388 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/4cbd74375ce1459584e1234a906e3578 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4cbd74375ce1459584e1234a906e3578 2024-11-23T05:41:55,391 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4cbd74375ce1459584e1234a906e3578, entries=150, sequenceid=219, filesize=11.9 K 2024-11-23T05:41:55,392 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/970c45ab244d4f42abf40a31412fe312 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/970c45ab244d4f42abf40a31412fe312 2024-11-23T05:41:55,395 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/970c45ab244d4f42abf40a31412fe312, entries=150, sequenceid=219, filesize=11.9 K 2024-11-23T05:41:55,396 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1265ms, sequenceid=219, compaction requested=true 2024-11-23T05:41:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:55,396 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=110}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=110 2024-11-23T05:41:55,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=110 2024-11-23T05:41:55,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=110, resume processing ppid=109 2024-11-23T05:41:55,399 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=110, ppid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.4180 sec 2024-11-23T05:41:55,400 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=109, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=109, table=TestAcidGuarantees in 1.4220 sec 2024-11-23T05:41:55,691 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:55,691 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:55,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:41:55,696 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b9719b4c4c314e51b207ec7bdf470ae9_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340514530/Put/seqid=0 2024-11-23T05:41:55,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742226_1402 (size=14794) 2024-11-23T05:41:55,711 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,712 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340575703, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,712 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340575707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,713 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340575708, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,713 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340575709, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340575816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340575816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,824 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,824 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,824 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340575816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,825 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:55,834 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:55,834 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340575826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,834 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:55,835 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,034 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340576026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,034 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,035 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340576036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340576036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,044 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,044 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,045 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340576036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,045 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,101 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:56,104 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b9719b4c4c314e51b207ec7bdf470ae9_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b9719b4c4c314e51b207ec7bdf470ae9_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:56,105 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/fd3a395d9c38446b9bc8c8e0482131f0, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:56,105 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/fd3a395d9c38446b9bc8c8e0482131f0 is 175, key is test_row_0/A:col10/1732340514530/Put/seqid=0 2024-11-23T05:41:56,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742227_1403 (size=39749) 2024-11-23T05:41:56,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=109 2024-11-23T05:41:56,127 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:56,127 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:41:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees 2024-11-23T05:41:56,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T05:41:56,129 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:41:56,129 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=111, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:41:56,129 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=112, ppid=111, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:41:56,236 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T05:41:56,281 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-23T05:41:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:56,281 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,281 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:56,282 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:56,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:56,350 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340576346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,351 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,354 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340576346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,355 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340576355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,361 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,361 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340576355, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,361 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,362 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,434 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-23T05:41:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,434 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] handler.RSProcedureHandler(58): pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:56,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=112 java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:56,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=112 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:41:56,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T05:41:56,510 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=245, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/fd3a395d9c38446b9bc8c8e0482131f0 2024-11-23T05:41:56,516 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/65c30e4477904a32b977f491c363ed00 is 50, key is test_row_0/B:col10/1732340514530/Put/seqid=0 2024-11-23T05:41:56,519 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742228_1404 (size=12151) 2024-11-23T05:41:56,519 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/65c30e4477904a32b977f491c363ed00 2024-11-23T05:41:56,533 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/464491121cd84297bd35be842a12b643 is 50, key is test_row_0/C:col10/1732340514530/Put/seqid=0 2024-11-23T05:41:56,536 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742229_1405 (size=12151) 2024-11-23T05:41:56,536 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/464491121cd84297bd35be842a12b643 2024-11-23T05:41:56,539 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/fd3a395d9c38446b9bc8c8e0482131f0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/fd3a395d9c38446b9bc8c8e0482131f0 2024-11-23T05:41:56,542 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/fd3a395d9c38446b9bc8c8e0482131f0, entries=200, sequenceid=245, filesize=38.8 K 2024-11-23T05:41:56,543 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/65c30e4477904a32b977f491c363ed00 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/65c30e4477904a32b977f491c363ed00 2024-11-23T05:41:56,546 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/65c30e4477904a32b977f491c363ed00, entries=150, sequenceid=245, filesize=11.9 K 2024-11-23T05:41:56,546 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/464491121cd84297bd35be842a12b643 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/464491121cd84297bd35be842a12b643 2024-11-23T05:41:56,550 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/464491121cd84297bd35be842a12b643, entries=150, sequenceid=245, filesize=11.9 K 2024-11-23T05:41:56,551 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 860ms, sequenceid=245, compaction requested=true 2024-11-23T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:56,551 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:56,551 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:56,551 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:56,552 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:56,552 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 133508 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:56,552 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/A is initiating minor compaction (all files) 2024-11-23T05:41:56,552 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/B is initiating minor compaction (all files) 2024-11-23T05:41:56,552 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/B in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,552 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/A in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,552 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/b1045438798d4a0dba3a1a96ba38e889, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/e12a6dd90156434dbe6d96ef75f80e13, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/411ced69e10344458dc8ffbf13431922, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/fd3a395d9c38446b9bc8c8e0482131f0] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=130.4 K 2024-11-23T05:41:56,552 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d8b9227a53b14c5e836b9bfe37f0dded, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d89e6911074d41bba2cc16177ded2808, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4cbd74375ce1459584e1234a906e3578, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/65c30e4477904a32b977f491c363ed00] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=47.9 K 2024-11-23T05:41:56,552 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=50.00 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,552 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/b1045438798d4a0dba3a1a96ba38e889, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/e12a6dd90156434dbe6d96ef75f80e13, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/411ced69e10344458dc8ffbf13431922, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/fd3a395d9c38446b9bc8c8e0482131f0] 2024-11-23T05:41:56,552 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d8b9227a53b14c5e836b9bfe37f0dded, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340510024 2024-11-23T05:41:56,552 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting b1045438798d4a0dba3a1a96ba38e889, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340510024 2024-11-23T05:41:56,553 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d89e6911074d41bba2cc16177ded2808, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732340510092 2024-11-23T05:41:56,553 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e12a6dd90156434dbe6d96ef75f80e13, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732340510092 2024-11-23T05:41:56,553 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4cbd74375ce1459584e1234a906e3578, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732340512278 2024-11-23T05:41:56,553 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 411ced69e10344458dc8ffbf13431922, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732340512278 2024-11-23T05:41:56,553 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 65c30e4477904a32b977f491c363ed00, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732340514530 2024-11-23T05:41:56,553 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting fd3a395d9c38446b9bc8c8e0482131f0, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732340514530 2024-11-23T05:41:56,560 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:56,561 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e202411230e096c39b23a461593767184731433c1_a9d4eac7c30b3b93cecf8ac4d22b4ff7 store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:56,562 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#B#compaction#338 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:56,562 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/3d7c36088428466ea03ccead9c7babf0 is 50, key is test_row_0/B:col10/1732340514530/Put/seqid=0 2024-11-23T05:41:56,564 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e202411230e096c39b23a461593767184731433c1_a9d4eac7c30b3b93cecf8ac4d22b4ff7, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:56,564 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230e096c39b23a461593767184731433c1_a9d4eac7c30b3b93cecf8ac4d22b4ff7 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:56,566 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742230_1406 (size=12731) 2024-11-23T05:41:56,568 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742231_1407 (size=4469) 2024-11-23T05:41:56,569 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#A#compaction#339 average throughput is 2.71 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:56,569 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8062874131b44049ab70101e91dacf4c is 175, key is test_row_0/A:col10/1732340514530/Put/seqid=0 2024-11-23T05:41:56,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742232_1408 (size=31685) 2024-11-23T05:41:56,587 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=112 2024-11-23T05:41:56,587 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,587 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T05:41:56,592 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236165452b7a3c4bdb814c32327630b1ab_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340515708/Put/seqid=0 2024-11-23T05:41:56,595 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742233_1409 (size=12304) 2024-11-23T05:41:56,598 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:56,601 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236165452b7a3c4bdb814c32327630b1ab_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236165452b7a3c4bdb814c32327630b1ab_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ff80eb579dad4736830faba81addb232, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:56,602 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ff80eb579dad4736830faba81addb232 is 175, key is test_row_0/A:col10/1732340515708/Put/seqid=0 2024-11-23T05:41:56,605 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742234_1410 (size=31105) 2024-11-23T05:41:56,606 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=255, memsize=15.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ff80eb579dad4736830faba81addb232 2024-11-23T05:41:56,611 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/f4cc491b3f284cc4a87f0b1e8f80b9e9 is 50, key is test_row_0/B:col10/1732340515708/Put/seqid=0 2024-11-23T05:41:56,615 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742235_1411 (size=12151) 2024-11-23T05:41:56,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T05:41:56,860 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:56,860 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. as already flushing 2024-11-23T05:41:56,922 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,922 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340576913, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,922 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,923 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 113 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340576914, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340576915, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,928 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:56,928 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340576916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:56,928 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:56,969 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/3d7c36088428466ea03ccead9c7babf0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/3d7c36088428466ea03ccead9c7babf0 2024-11-23T05:41:56,973 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/B of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 3d7c36088428466ea03ccead9c7babf0(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:56,973 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:56,973 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, priority=12, startTime=1732340516551; duration=0sec 2024-11-23T05:41:56,973 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:41:56,973 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:B 2024-11-23T05:41:56,973 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:41:56,974 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49048 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:41:56,974 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): a9d4eac7c30b3b93cecf8ac4d22b4ff7/C is initiating minor compaction (all files) 2024-11-23T05:41:56,974 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of a9d4eac7c30b3b93cecf8ac4d22b4ff7/C in TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:56,974 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/42cb7fb3b2c44009a5e8f6440682ea1f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/4afe3ec0a46a42cd94176e07cd881a07, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/970c45ab244d4f42abf40a31412fe312, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/464491121cd84297bd35be842a12b643] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp, totalSize=47.9 K 2024-11-23T05:41:56,975 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 42cb7fb3b2c44009a5e8f6440682ea1f, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=180, earliestPutTs=1732340510024 2024-11-23T05:41:56,975 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4afe3ec0a46a42cd94176e07cd881a07, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=207, earliestPutTs=1732340510092 2024-11-23T05:41:56,975 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 970c45ab244d4f42abf40a31412fe312, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=219, earliestPutTs=1732340512278 2024-11-23T05:41:56,975 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 464491121cd84297bd35be842a12b643, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732340514530 2024-11-23T05:41:56,975 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8062874131b44049ab70101e91dacf4c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8062874131b44049ab70101e91dacf4c 2024-11-23T05:41:56,980 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/A of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into 8062874131b44049ab70101e91dacf4c(size=30.9 K), total size for store is 30.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:56,980 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:56,980 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, priority=12, startTime=1732340516551; duration=0sec 2024-11-23T05:41:56,980 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:56,980 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:A 2024-11-23T05:41:56,981 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): a9d4eac7c30b3b93cecf8ac4d22b4ff7#C#compaction#342 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 50.00 MB/second 2024-11-23T05:41:56,981 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/a9c2a0c5e2024ab88f58c975c63fd405 is 50, key is test_row_0/C:col10/1732340514530/Put/seqid=0 2024-11-23T05:41:56,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742236_1412 (size=12731) 2024-11-23T05:41:57,015 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/f4cc491b3f284cc4a87f0b1e8f80b9e9 2024-11-23T05:41:57,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/50e1ce5ebe35480296b52aa3a43656ff is 50, key is test_row_0/C:col10/1732340515708/Put/seqid=0 2024-11-23T05:41:57,024 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742237_1413 (size=12151) 2024-11-23T05:41:57,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340577026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,031 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 114 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340577036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340577036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,043 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340577036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,043 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,166 INFO [Thread-1557 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1557. 2024-11-23T05:41:57,166 DEBUG [Thread-1557 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,166 INFO [Thread-1551 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1551. 2024-11-23T05:41:57,166 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,166 DEBUG [Thread-1551 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,166 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,166 DEBUG [Thread-1551 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,166 DEBUG [Thread-1557 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,166 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,168 INFO [Thread-1555 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1555. 2024-11-23T05:41:57,168 DEBUG [Thread-1555 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,168 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,168 DEBUG [Thread-1555 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,168 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,174 INFO [Thread-1553 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1553. 2024-11-23T05:41:57,174 DEBUG [Thread-1553 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,174 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,174 DEBUG [Thread-1553 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,174 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,175 INFO [Thread-1559 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1559. 2024-11-23T05:41:57,175 DEBUG [Thread-1559 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,175 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,175 DEBUG [Thread-1559 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,175 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,237 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,237 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41680 deadline: 1732340577236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,237 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,247 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,247 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 115 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41698 deadline: 1732340577247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,247 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41610 deadline: 1732340577247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,248 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,248 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:41608 deadline: 1732340577247, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,248 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:41:57,249 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:41:57,267 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T05:41:57,302 DEBUG [regionserver/9abfc7c2569e:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.07692307692307693, tune throughput to 53.85 MB/second 2024-11-23T05:41:57,305 WARN [regionserver/9abfc7c2569e:0.Chore.1 {}] hbase.ExecutorStatusChore(69): RS_COMPACTED_FILES_DISCHARGER's size info, queued: 5, running: 0 2024-11-23T05:41:57,309 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c3ce8d689ec440cabceb6e94d4ec7540, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/09e77500c9604211b89ef0af56a3cabb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/4b964a1f09a74979abce8e822152e3e8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2bc1432dd0e14281a549510acb09ae49, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2eb4412325ac416a91e71e51924d5797, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8d0e368e4b3541e09a1804ab8e2806e1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/100506c4cf2b4ff383cc5832bf071d21, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/6f48876899be422a91a70db0f19cb7a0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/55329dd72dae4aeea09ac5f2aed95888, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/a198d0b2e32046448f4c270336d08a84, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ebe7cad7685f450a9730bcfd88a69001, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/7a3684fbbc7f4a7db477a3b9e248ade8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ae1ae9397cf44ce6be8e89f26808675c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/b1045438798d4a0dba3a1a96ba38e889, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/e12a6dd90156434dbe6d96ef75f80e13, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/411ced69e10344458dc8ffbf13431922, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/fd3a395d9c38446b9bc8c8e0482131f0] to archive 2024-11-23T05:41:57,310 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:57,311 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c3ce8d689ec440cabceb6e94d4ec7540 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c3ce8d689ec440cabceb6e94d4ec7540 2024-11-23T05:41:57,312 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/09e77500c9604211b89ef0af56a3cabb to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/09e77500c9604211b89ef0af56a3cabb 2024-11-23T05:41:57,313 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/4b964a1f09a74979abce8e822152e3e8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/4b964a1f09a74979abce8e822152e3e8 2024-11-23T05:41:57,314 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2bc1432dd0e14281a549510acb09ae49 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2bc1432dd0e14281a549510acb09ae49 2024-11-23T05:41:57,315 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2eb4412325ac416a91e71e51924d5797 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/2eb4412325ac416a91e71e51924d5797 2024-11-23T05:41:57,316 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8d0e368e4b3541e09a1804ab8e2806e1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8d0e368e4b3541e09a1804ab8e2806e1 2024-11-23T05:41:57,317 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/100506c4cf2b4ff383cc5832bf071d21 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/100506c4cf2b4ff383cc5832bf071d21 2024-11-23T05:41:57,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/6f48876899be422a91a70db0f19cb7a0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/6f48876899be422a91a70db0f19cb7a0 2024-11-23T05:41:57,318 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/55329dd72dae4aeea09ac5f2aed95888 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/55329dd72dae4aeea09ac5f2aed95888 2024-11-23T05:41:57,319 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/a198d0b2e32046448f4c270336d08a84 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/a198d0b2e32046448f4c270336d08a84 2024-11-23T05:41:57,320 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ebe7cad7685f450a9730bcfd88a69001 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ebe7cad7685f450a9730bcfd88a69001 2024-11-23T05:41:57,321 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/7a3684fbbc7f4a7db477a3b9e248ade8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/7a3684fbbc7f4a7db477a3b9e248ade8 2024-11-23T05:41:57,322 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ae1ae9397cf44ce6be8e89f26808675c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ae1ae9397cf44ce6be8e89f26808675c 2024-11-23T05:41:57,323 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/b1045438798d4a0dba3a1a96ba38e889 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/b1045438798d4a0dba3a1a96ba38e889 2024-11-23T05:41:57,324 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/e12a6dd90156434dbe6d96ef75f80e13 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/e12a6dd90156434dbe6d96ef75f80e13 2024-11-23T05:41:57,325 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/411ced69e10344458dc8ffbf13431922 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/411ced69e10344458dc8ffbf13431922 2024-11-23T05:41:57,326 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/fd3a395d9c38446b9bc8c8e0482131f0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/fd3a395d9c38446b9bc8c8e0482131f0 2024-11-23T05:41:57,329 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/eed7c0199cb44e9aad48f3ad4a31df41, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/0cabf826ea0041c9ac656f88b8204d73, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/c4cfba76357a46b18171dc2539f18c40, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/9f81c4f8f79b436f9ca8e0dfc5721174, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1a5831a6ad684128bdb6aae39d7deefa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4e923494fe2d42fb9a4f9e537582055b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2b404e3281654d828b76cf8e6c26f184, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/ee9e4c34db17499ea0d39a70ee6e4ba1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1dfd00bce5d748e591a9be819129ec87, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2d13d5ee5c1f4b8a9b07d85714266b3d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/874bcf5d914a4908ae939e087b515bcf, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7d432e597ef44bebb7711f4a390f99be, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d8b9227a53b14c5e836b9bfe37f0dded, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7ae28256a76241558fe737a73239ebaa, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d89e6911074d41bba2cc16177ded2808, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4cbd74375ce1459584e1234a906e3578, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/65c30e4477904a32b977f491c363ed00] to archive 2024-11-23T05:41:57,330 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:57,331 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/eed7c0199cb44e9aad48f3ad4a31df41 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/eed7c0199cb44e9aad48f3ad4a31df41 2024-11-23T05:41:57,332 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/0cabf826ea0041c9ac656f88b8204d73 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/0cabf826ea0041c9ac656f88b8204d73 2024-11-23T05:41:57,333 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/c4cfba76357a46b18171dc2539f18c40 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/c4cfba76357a46b18171dc2539f18c40 2024-11-23T05:41:57,333 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/9f81c4f8f79b436f9ca8e0dfc5721174 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/9f81c4f8f79b436f9ca8e0dfc5721174 2024-11-23T05:41:57,334 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1a5831a6ad684128bdb6aae39d7deefa to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1a5831a6ad684128bdb6aae39d7deefa 2024-11-23T05:41:57,335 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4e923494fe2d42fb9a4f9e537582055b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4e923494fe2d42fb9a4f9e537582055b 2024-11-23T05:41:57,336 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2b404e3281654d828b76cf8e6c26f184 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2b404e3281654d828b76cf8e6c26f184 2024-11-23T05:41:57,337 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/ee9e4c34db17499ea0d39a70ee6e4ba1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/ee9e4c34db17499ea0d39a70ee6e4ba1 2024-11-23T05:41:57,338 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1dfd00bce5d748e591a9be819129ec87 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/1dfd00bce5d748e591a9be819129ec87 2024-11-23T05:41:57,339 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2d13d5ee5c1f4b8a9b07d85714266b3d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/2d13d5ee5c1f4b8a9b07d85714266b3d 2024-11-23T05:41:57,340 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/874bcf5d914a4908ae939e087b515bcf to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/874bcf5d914a4908ae939e087b515bcf 2024-11-23T05:41:57,341 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7d432e597ef44bebb7711f4a390f99be to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7d432e597ef44bebb7711f4a390f99be 2024-11-23T05:41:57,342 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d8b9227a53b14c5e836b9bfe37f0dded to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d8b9227a53b14c5e836b9bfe37f0dded 2024-11-23T05:41:57,343 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7ae28256a76241558fe737a73239ebaa to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/7ae28256a76241558fe737a73239ebaa 2024-11-23T05:41:57,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d89e6911074d41bba2cc16177ded2808 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/d89e6911074d41bba2cc16177ded2808 2024-11-23T05:41:57,344 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4cbd74375ce1459584e1234a906e3578 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/4cbd74375ce1459584e1234a906e3578 2024-11-23T05:41:57,345 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/65c30e4477904a32b977f491c363ed00 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/65c30e4477904a32b977f491c363ed00 2024-11-23T05:41:57,347 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3b400278b0444cf98406c79d80907ffc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1b7d4d104e9e4dd8a787df4319ae87c7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ca007131bb79469ba38e29827efeddca, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ee7ca6edbba24a788e18b20c8bc5e663, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/8421f729344a4f07940f3ce683f904b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/74083749b70f4b8187fdcc67d95f4d37, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1ccd2eccb2194b28aea511d0062bdb75, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/6c7d9ff858404c92827b678f5a6ab28c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/b49a963d80db4835877ef63bebf5adf5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/2ea81b50fd1e460bacbdb33768086661, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c6b15529b2fb49c49277c1d234bb5550, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3aebc5abe39a4e2987b9f96eef7b325d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e15e8592e92741599f30547e70ee8799] to archive 2024-11-23T05:41:57,348 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:41:57,349 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3b400278b0444cf98406c79d80907ffc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3b400278b0444cf98406c79d80907ffc 2024-11-23T05:41:57,350 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1b7d4d104e9e4dd8a787df4319ae87c7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1b7d4d104e9e4dd8a787df4319ae87c7 2024-11-23T05:41:57,351 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ca007131bb79469ba38e29827efeddca to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ca007131bb79469ba38e29827efeddca 2024-11-23T05:41:57,352 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ee7ca6edbba24a788e18b20c8bc5e663 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/ee7ca6edbba24a788e18b20c8bc5e663 2024-11-23T05:41:57,353 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/8421f729344a4f07940f3ce683f904b7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/8421f729344a4f07940f3ce683f904b7 2024-11-23T05:41:57,353 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/74083749b70f4b8187fdcc67d95f4d37 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/74083749b70f4b8187fdcc67d95f4d37 2024-11-23T05:41:57,354 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1ccd2eccb2194b28aea511d0062bdb75 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/1ccd2eccb2194b28aea511d0062bdb75 2024-11-23T05:41:57,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/6c7d9ff858404c92827b678f5a6ab28c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/6c7d9ff858404c92827b678f5a6ab28c 2024-11-23T05:41:57,355 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/b49a963d80db4835877ef63bebf5adf5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/b49a963d80db4835877ef63bebf5adf5 2024-11-23T05:41:57,356 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/2ea81b50fd1e460bacbdb33768086661 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/2ea81b50fd1e460bacbdb33768086661 2024-11-23T05:41:57,357 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c6b15529b2fb49c49277c1d234bb5550 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c6b15529b2fb49c49277c1d234bb5550 2024-11-23T05:41:57,357 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3aebc5abe39a4e2987b9f96eef7b325d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/3aebc5abe39a4e2987b9f96eef7b325d 2024-11-23T05:41:57,358 DEBUG [RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 {event_type=RS_COMPACTED_FILES_DISCHARGER}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e15e8592e92741599f30547e70ee8799 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e15e8592e92741599f30547e70ee8799 2024-11-23T05:41:57,388 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/a9c2a0c5e2024ab88f58c975c63fd405 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/a9c2a0c5e2024ab88f58c975c63fd405 2024-11-23T05:41:57,392 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in a9d4eac7c30b3b93cecf8ac4d22b4ff7/C of a9d4eac7c30b3b93cecf8ac4d22b4ff7 into a9c2a0c5e2024ab88f58c975c63fd405(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:41:57,392 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:57,392 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., storeName=a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, priority=12, startTime=1732340516551; duration=0sec 2024-11-23T05:41:57,392 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:57,392 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:C 2024-11-23T05:41:57,425 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=255 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/50e1ce5ebe35480296b52aa3a43656ff 2024-11-23T05:41:57,428 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/ff80eb579dad4736830faba81addb232 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ff80eb579dad4736830faba81addb232 2024-11-23T05:41:57,432 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ff80eb579dad4736830faba81addb232, entries=150, sequenceid=255, filesize=30.4 K 2024-11-23T05:41:57,433 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/f4cc491b3f284cc4a87f0b1e8f80b9e9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/f4cc491b3f284cc4a87f0b1e8f80b9e9 2024-11-23T05:41:57,436 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/f4cc491b3f284cc4a87f0b1e8f80b9e9, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T05:41:57,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/50e1ce5ebe35480296b52aa3a43656ff as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/50e1ce5ebe35480296b52aa3a43656ff 2024-11-23T05:41:57,441 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/50e1ce5ebe35480296b52aa3a43656ff, entries=150, sequenceid=255, filesize=11.9 K 2024-11-23T05:41:57,442 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 854ms, sequenceid=255, compaction requested=false 2024-11-23T05:41:57,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:57,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:57,442 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=112}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=112 2024-11-23T05:41:57,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=112 2024-11-23T05:41:57,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=112, resume processing ppid=111 2024-11-23T05:41:57,444 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=112, ppid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3140 sec 2024-11-23T05:41:57,445 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=111, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=111, table=TestAcidGuarantees in 1.3170 sec 2024-11-23T05:41:57,549 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:57,550 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:41:57,550 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T05:41:57,550 INFO [Thread-1542 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1542. 2024-11-23T05:41:57,550 DEBUG [Thread-1542 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,550 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,551 DEBUG [Thread-1542 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,551 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,559 INFO [Thread-1544 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1544. 2024-11-23T05:41:57,559 DEBUG [Thread-1544 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,559 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,559 INFO [Thread-1546 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1546. 2024-11-23T05:41:57,559 INFO [Thread-1548 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1548. 2024-11-23T05:41:57,559 DEBUG [Thread-1548 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,559 DEBUG [Thread-1546 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,559 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,559 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,559 DEBUG [Thread-1544 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,559 DEBUG [Thread-1546 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,559 DEBUG [Thread-1548 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,559 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,561 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112397ec84c907c14db89c0b53adaaebeff3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340516913/Put/seqid=0 2024-11-23T05:41:57,565 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742238_1414 (size=12454) 2024-11-23T05:41:57,801 INFO [Thread-1540 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1540. 2024-11-23T05:41:57,802 DEBUG [Thread-1540 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:41:57,802 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,802 DEBUG [Thread-1540 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:57,802 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:57,967 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:57,976 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112397ec84c907c14db89c0b53adaaebeff3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112397ec84c907c14db89c0b53adaaebeff3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:57,977 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8a104f43fc3f4b52af27114bdf1e0c37, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:57,977 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8a104f43fc3f4b52af27114bdf1e0c37 is 175, key is test_row_0/A:col10/1732340516913/Put/seqid=0 2024-11-23T05:41:57,980 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742239_1415 (size=31255) 2024-11-23T05:41:58,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=111 2024-11-23T05:41:58,277 INFO [RPCClient-NioEventLoopGroup-4-8 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 13 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 79 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 44 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1547 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4637 rows 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1553 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4652 rows 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1542 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4622 rows 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1546 2024-11-23T05:41:58,277 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4632 rows 2024-11-23T05:41:58,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1547 2024-11-23T05:41:58,278 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4637 rows 2024-11-23T05:41:58,278 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:41:58,278 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobScanAtomicity(AcidGuaranteesTestBase.java:127) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:41:58,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:58,278 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:41:58,278 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:41:58,280 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T05:41:58,282 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=113, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T05:41:58,285 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T05:41:58,286 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340518286"}]},"ts":"1732340518286"} 2024-11-23T05:41:58,289 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T05:41:58,289 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T05:41:58,290 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=114, ppid=113, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:41:58,291 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, UNASSIGN}] 2024-11-23T05:41:58,292 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, UNASSIGN 2024-11-23T05:41:58,293 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:41:58,294 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=115, ppid=114, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, UNASSIGN because future has completed 2024-11-23T05:41:58,294 DEBUG [PEWorker-3 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:41:58,294 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:41:58,382 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=285, memsize=53.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8a104f43fc3f4b52af27114bdf1e0c37 2024-11-23T05:41:58,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T05:41:58,398 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/8ad6db5da1f4491dab2606532ec49bdf is 50, key is test_row_0/B:col10/1732340516913/Put/seqid=0 2024-11-23T05:41:58,401 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742240_1416 (size=12301) 2024-11-23T05:41:58,446 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(122): Close a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:58,446 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:41:58,446 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1722): Closing a9d4eac7c30b3b93cecf8ac4d22b4ff7, disabling compactions & flushes 2024-11-23T05:41:58,447 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1993): waiting for 0 compactions & cache flush to complete for region TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:58,607 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T05:41:58,802 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/8ad6db5da1f4491dab2606532ec49bdf 2024-11-23T05:41:58,814 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/e804436161594734b8308d42083ff96e is 50, key is test_row_0/C:col10/1732340516913/Put/seqid=0 2024-11-23T05:41:58,818 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742241_1417 (size=12301) 2024-11-23T05:41:58,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T05:41:59,220 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=285 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/e804436161594734b8308d42083ff96e 2024-11-23T05:41:59,231 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/8a104f43fc3f4b52af27114bdf1e0c37 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8a104f43fc3f4b52af27114bdf1e0c37 2024-11-23T05:41:59,235 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8a104f43fc3f4b52af27114bdf1e0c37, entries=150, sequenceid=285, filesize=30.5 K 2024-11-23T05:41:59,236 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/8ad6db5da1f4491dab2606532ec49bdf as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/8ad6db5da1f4491dab2606532ec49bdf 2024-11-23T05:41:59,239 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/8ad6db5da1f4491dab2606532ec49bdf, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T05:41:59,240 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/e804436161594734b8308d42083ff96e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e804436161594734b8308d42083ff96e 2024-11-23T05:41:59,243 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e804436161594734b8308d42083ff96e, entries=150, sequenceid=285, filesize=12.0 K 2024-11-23T05:41:59,243 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=26.84 KB/27480 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1693ms, sequenceid=285, compaction requested=true 2024-11-23T05:41:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: 2024-11-23T05:41:59,244 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:41:59,244 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:59,244 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. because compaction request was cancelled 2024-11-23T05:41:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:41:59,244 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. after waiting 0 ms 2024-11-23T05:41:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:59,244 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:A 2024-11-23T05:41:59,244 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. because compaction request was cancelled 2024-11-23T05:41:59,244 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:41:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store a9d4eac7c30b3b93cecf8ac4d22b4ff7:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:41:59,244 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:41:59,244 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit(450): Not compacting TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. because compaction request was cancelled 2024-11-23T05:41:59,244 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:B 2024-11-23T05:41:59,244 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: a9d4eac7c30b3b93cecf8ac4d22b4ff7:C 2024-11-23T05:41:59,244 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(2902): Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7 3/3 column families, dataSize=26.84 KB heapSize=71.06 KB 2024-11-23T05:41:59,248 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123579148b73bfe421da949c12d11bdbed3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 is 50, key is test_row_0/A:col10/1732340517557/Put/seqid=0 2024-11-23T05:41:59,251 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742242_1418 (size=12454) 2024-11-23T05:41:59,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T05:41:59,653 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:41:59,663 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123579148b73bfe421da949c12d11bdbed3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123579148b73bfe421da949c12d11bdbed3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:41:59,664 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c9b7628359e64daa8e773cacd8348b33, store: [table=TestAcidGuarantees family=A region=a9d4eac7c30b3b93cecf8ac4d22b4ff7] 2024-11-23T05:41:59,665 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c9b7628359e64daa8e773cacd8348b33 is 175, key is test_row_0/A:col10/1732340517557/Put/seqid=0 2024-11-23T05:41:59,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742243_1419 (size=31255) 2024-11-23T05:42:00,071 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=8.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c9b7628359e64daa8e773cacd8348b33 2024-11-23T05:42:00,082 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/23fe4f2664a543288caee75445bac057 is 50, key is test_row_0/B:col10/1732340517557/Put/seqid=0 2024-11-23T05:42:00,086 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742244_1420 (size=12301) 2024-11-23T05:42:00,437 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T05:42:00,488 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/23fe4f2664a543288caee75445bac057 2024-11-23T05:42:00,501 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/c9e4d6f2c03c4e1e90ed63afc09cc36c is 50, key is test_row_0/C:col10/1732340517557/Put/seqid=0 2024-11-23T05:42:00,504 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742245_1421 (size=12301) 2024-11-23T05:42:00,906 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=8.95 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/c9e4d6f2c03c4e1e90ed63afc09cc36c 2024-11-23T05:42:00,915 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/A/c9b7628359e64daa8e773cacd8348b33 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c9b7628359e64daa8e773cacd8348b33 2024-11-23T05:42:00,920 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c9b7628359e64daa8e773cacd8348b33, entries=150, sequenceid=292, filesize=30.5 K 2024-11-23T05:42:00,921 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/B/23fe4f2664a543288caee75445bac057 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/23fe4f2664a543288caee75445bac057 2024-11-23T05:42:00,926 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/23fe4f2664a543288caee75445bac057, entries=150, sequenceid=292, filesize=12.0 K 2024-11-23T05:42:00,927 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/.tmp/C/c9e4d6f2c03c4e1e90ed63afc09cc36c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c9e4d6f2c03c4e1e90ed63afc09cc36c 2024-11-23T05:42:00,930 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c9e4d6f2c03c4e1e90ed63afc09cc36c, entries=150, sequenceid=292, filesize=12.0 K 2024-11-23T05:42:00,931 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(3140): Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1687ms, sequenceid=292, compaction requested=true 2024-11-23T05:42:00,932 DEBUG [StoreCloser-TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/42cb7fb3b2c44009a5e8f6440682ea1f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/4afe3ec0a46a42cd94176e07cd881a07, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/970c45ab244d4f42abf40a31412fe312, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/464491121cd84297bd35be842a12b643] to archive 2024-11-23T05:42:00,933 DEBUG [StoreCloser-TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:42:00,934 DEBUG [StoreCloser-TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/42cb7fb3b2c44009a5e8f6440682ea1f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/42cb7fb3b2c44009a5e8f6440682ea1f 2024-11-23T05:42:00,935 DEBUG [StoreCloser-TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/4afe3ec0a46a42cd94176e07cd881a07 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/4afe3ec0a46a42cd94176e07cd881a07 2024-11-23T05:42:00,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/970c45ab244d4f42abf40a31412fe312 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/970c45ab244d4f42abf40a31412fe312 2024-11-23T05:42:00,936 DEBUG [StoreCloser-TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/464491121cd84297bd35be842a12b643 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/464491121cd84297bd35be842a12b643 2024-11-23T05:42:00,940 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/recovered.edits/295.seqid, newMaxSeqId=295, maxSeqId=4 2024-11-23T05:42:00,940 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. 2024-11-23T05:42:00,940 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] regionserver.HRegion(1676): Region close journal for a9d4eac7c30b3b93cecf8ac4d22b4ff7: Waiting for close lock at 1732340518446Running coprocessor pre-close hooks at 1732340518446Disabling compacts and flushes for region at 1732340518446Disabling writes for close at 1732340519244 (+798 ms)Obtaining lock to block concurrent updates at 1732340519244Preparing flush snapshotting stores in a9d4eac7c30b3b93cecf8ac4d22b4ff7 at 1732340519244Finished memstore snapshotting TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7., syncing WAL and waiting on mvcc, flushsize=dataSize=27480, getHeapSize=72720, getOffHeapSize=0, getCellsCount=600 at 1732340519244Flushing stores of TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7. at 1732340519244Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/A: creating writer at 1732340519245 (+1 ms)Flushing mob file a9d4eac7c30b3b93cecf8ac4d22b4ff7/A: appending metadata at 1732340519248 (+3 ms)Flushing mob file a9d4eac7c30b3b93cecf8ac4d22b4ff7/A: closing flushed file at 1732340519248Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/A: appending metadata at 1732340519664 (+416 ms)Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/A: closing flushed file at 1732340519664Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/B: creating writer at 1732340520079 (+415 ms)Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/B: appending metadata at 1732340520082 (+3 ms)Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/B: closing flushed file at 1732340520082Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/C: creating writer at 1732340520497 (+415 ms)Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/C: appending metadata at 1732340520500 (+3 ms)Flushing a9d4eac7c30b3b93cecf8ac4d22b4ff7/C: closing flushed file at 1732340520500Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5772ada4: reopening flushed file at 1732340520914 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@451f7d42: reopening flushed file at 1732340520921 (+7 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3991bd7e: reopening flushed file at 1732340520926 (+5 ms)Finished flush of dataSize ~26.84 KB/27480, heapSize ~71.02 KB/72720, currentSize=0 B/0 for a9d4eac7c30b3b93cecf8ac4d22b4ff7 in 1687ms, sequenceid=292, compaction requested=true at 1732340520931 (+5 ms)Writing region close event to WAL at 1732340520937 (+6 ms)Running coprocessor post-close hooks at 1732340520940 (+3 ms)Closed at 1732340520940 2024-11-23T05:42:00,942 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=116}] handler.UnassignRegionHandler(157): Closed a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:00,942 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=115 updating hbase:meta row=a9d4eac7c30b3b93cecf8ac4d22b4ff7, regionState=CLOSED 2024-11-23T05:42:00,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=116, ppid=115, state=RUNNABLE, hasLock=false; CloseRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:42:00,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=116, resume processing ppid=115 2024-11-23T05:42:00,945 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=116, ppid=115, state=SUCCESS, hasLock=false; CloseRegionProcedure a9d4eac7c30b3b93cecf8ac4d22b4ff7, server=9abfc7c2569e,45719,1732340394120 in 2.6500 sec 2024-11-23T05:42:00,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=115, resume processing ppid=114 2024-11-23T05:42:00,947 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=115, ppid=114, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=a9d4eac7c30b3b93cecf8ac4d22b4ff7, UNASSIGN in 2.6540 sec 2024-11-23T05:42:00,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=114, resume processing ppid=113 2024-11-23T05:42:00,948 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=114, ppid=113, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 2.6580 sec 2024-11-23T05:42:00,949 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340520949"}]},"ts":"1732340520949"} 2024-11-23T05:42:00,950 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T05:42:00,950 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T05:42:00,952 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=113, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 2.6710 sec 2024-11-23T05:42:02,457 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=113 2024-11-23T05:42:02,457 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:02,458 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T05:42:02,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:02,498 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=117, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:02,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T05:42:02,500 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=117, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:02,503 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,507 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/recovered.edits] 2024-11-23T05:42:02,510 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8062874131b44049ab70101e91dacf4c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8062874131b44049ab70101e91dacf4c 2024-11-23T05:42:02,511 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8a104f43fc3f4b52af27114bdf1e0c37 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/8a104f43fc3f4b52af27114bdf1e0c37 2024-11-23T05:42:02,513 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c9b7628359e64daa8e773cacd8348b33 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/c9b7628359e64daa8e773cacd8348b33 2024-11-23T05:42:02,514 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ff80eb579dad4736830faba81addb232 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/A/ff80eb579dad4736830faba81addb232 2024-11-23T05:42:02,517 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/23fe4f2664a543288caee75445bac057 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/23fe4f2664a543288caee75445bac057 2024-11-23T05:42:02,518 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/3d7c36088428466ea03ccead9c7babf0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/3d7c36088428466ea03ccead9c7babf0 2024-11-23T05:42:02,520 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/8ad6db5da1f4491dab2606532ec49bdf to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/8ad6db5da1f4491dab2606532ec49bdf 2024-11-23T05:42:02,521 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/f4cc491b3f284cc4a87f0b1e8f80b9e9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/B/f4cc491b3f284cc4a87f0b1e8f80b9e9 2024-11-23T05:42:02,524 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/50e1ce5ebe35480296b52aa3a43656ff to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/50e1ce5ebe35480296b52aa3a43656ff 2024-11-23T05:42:02,525 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/a9c2a0c5e2024ab88f58c975c63fd405 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/a9c2a0c5e2024ab88f58c975c63fd405 2024-11-23T05:42:02,527 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c9e4d6f2c03c4e1e90ed63afc09cc36c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/c9e4d6f2c03c4e1e90ed63afc09cc36c 2024-11-23T05:42:02,529 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e804436161594734b8308d42083ff96e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/C/e804436161594734b8308d42083ff96e 2024-11-23T05:42:02,532 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/recovered.edits/295.seqid to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7/recovered.edits/295.seqid 2024-11-23T05:42:02,533 DEBUG [HFileArchiver-4 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,533 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T05:42:02,533 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T05:42:02,534 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T05:42:02,538 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112306e7a4e0de624408a99f09da2a77d482_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112306e7a4e0de624408a99f09da2a77d482_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,540 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112356de2d9f84e44c6db7ba8fe0f58f7a56_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112356de2d9f84e44c6db7ba8fe0f58f7a56_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,542 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123579148b73bfe421da949c12d11bdbed3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123579148b73bfe421da949c12d11bdbed3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,544 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236165452b7a3c4bdb814c32327630b1ab_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236165452b7a3c4bdb814c32327630b1ab_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,545 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123639465f9c84b47d79fbeaf1f4485d301_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123639465f9c84b47d79fbeaf1f4485d301_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,546 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236bfc4389b057424792f0ac981af6fc68_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236bfc4389b057424792f0ac981af6fc68_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,547 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112397ec84c907c14db89c0b53adaaebeff3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112397ec84c907c14db89c0b53adaaebeff3_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,548 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a9f1da1d9200411299446153a0ba2497_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123a9f1da1d9200411299446153a0ba2497_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,549 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123aff2aeedde4f47a3981834c1ffa80221_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123aff2aeedde4f47a3981834c1ffa80221_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,550 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b9719b4c4c314e51b207ec7bdf470ae9_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b9719b4c4c314e51b207ec7bdf470ae9_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,551 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dbb992df02504a60a10c4bf6a04c8a03_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123dbb992df02504a60a10c4bf6a04c8a03_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,552 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ea0e1d592e9548c58e6b42d26f257e51_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ea0e1d592e9548c58e6b42d26f257e51_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,553 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eef412bc42f64a068026f6c03b80d1b8_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123eef412bc42f64a068026f6c03b80d1b8_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,554 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123efdefb043fa54b898989f633b4efd25f_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123efdefb043fa54b898989f633b4efd25f_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,555 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fe5ec454a9ce4242b10971b83e7ea73a_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123fe5ec454a9ce4242b10971b83e7ea73a_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,556 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ff836a45fa4541859402cf461005dc0d_a9d4eac7c30b3b93cecf8ac4d22b4ff7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123ff836a45fa4541859402cf461005dc0d_a9d4eac7c30b3b93cecf8ac4d22b4ff7 2024-11-23T05:42:02,556 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T05:42:02,558 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=117, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:02,559 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T05:42:02,561 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T05:42:02,562 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=117, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:02,562 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T05:42:02,562 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732340522562"}]},"ts":"9223372036854775807"} 2024-11-23T05:42:02,563 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T05:42:02,563 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => a9d4eac7c30b3b93cecf8ac4d22b4ff7, NAME => 'TestAcidGuarantees,,1732340494021.a9d4eac7c30b3b93cecf8ac4d22b4ff7.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T05:42:02,563 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T05:42:02,564 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732340522563"}]},"ts":"9223372036854775807"} 2024-11-23T05:42:02,565 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T05:42:02,566 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=117, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:02,566 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=117, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 108 msec 2024-11-23T05:42:02,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=117 2024-11-23T05:42:02,606 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T05:42:02,606 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:02,614 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobScanAtomicity Thread=243 (was 240) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935771990_22 at /127.0.0.1:39782 [Waiting for operation #194] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: HFileArchiver-4 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.awaitNanos(AbstractQueuedSynchronizer.java:1674) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.poll(LinkedBlockingQueue.java:460) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1061) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: RS_COMPACTED_FILES_DISCHARGER-regionserver/9abfc7c2569e:0-0 java.base@17.0.11/jdk.internal.misc.Unsafe.park(Native Method) java.base@17.0.11/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionNode.block(AbstractQueuedSynchronizer.java:506) java.base@17.0.11/java.util.concurrent.ForkJoinPool.unmanagedBlock(ForkJoinPool.java:3465) java.base@17.0.11/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3436) java.base@17.0.11/java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:1625) java.base@17.0.11/java.util.concurrent.LinkedBlockingQueue.take(LinkedBlockingQueue.java:435) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1062) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1122) java.base@17.0.11/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935771990_22 at /127.0.0.1:39708 [Waiting for operation #792] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-24366377_22 at /127.0.0.1:49976 [Waiting for operation #1068] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_1935771990_22 at /127.0.0.1:39698 [Waiting for operation #792] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) Potentially hanging thread: DataXceiver for client DFSClient_NONMAPREDUCE_-24366377_22 at /127.0.0.1:39696 [Waiting for operation #794] java.base@17.0.11/sun.nio.ch.EPoll.wait(Native Method) java.base@17.0.11/sun.nio.ch.EPollSelectorImpl.doSelect(EPollSelectorImpl.java:118) java.base@17.0.11/sun.nio.ch.SelectorImpl.lockAndDoSelect(SelectorImpl.java:129) java.base@17.0.11/sun.nio.ch.SelectorImpl.select(SelectorImpl.java:141) app//org.apache.hadoop.net.SocketIOWithTimeout$SelectorPool.select(SocketIOWithTimeout.java:335) app//org.apache.hadoop.net.SocketIOWithTimeout.doIO(SocketIOWithTimeout.java:156) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:161) app//org.apache.hadoop.net.SocketInputStream.read(SocketInputStream.java:131) java.base@17.0.11/java.io.BufferedInputStream.fill(BufferedInputStream.java:244) java.base@17.0.11/java.io.BufferedInputStream.read(BufferedInputStream.java:263) java.base@17.0.11/java.io.DataInputStream.readUnsignedShort(DataInputStream.java:334) java.base@17.0.11/java.io.DataInputStream.readShort(DataInputStream.java:312) app//org.apache.hadoop.hdfs.protocol.datatransfer.Receiver.readOp(Receiver.java:72) app//org.apache.hadoop.hdfs.server.datanode.DataXceiver.run(DataXceiver.java:273) java.base@17.0.11/java.lang.Thread.run(Thread.java:840) - Thread LEAK? -, OpenFileDescriptor=459 (was 455) - OpenFileDescriptor LEAK? -, MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=445 (was 522), ProcessCount=11 (was 11), AvailableMemoryMB=8666 (was 8721) 2024-11-23T05:42:02,621 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=243, OpenFileDescriptor=459, MaxFileDescriptor=1048576, SystemLoadAverage=445, ProcessCount=11, AvailableMemoryMB=8665 2024-11-23T05:42:02,622 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:42:02,622 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:42:02,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:02,624 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:42:02,625 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:02,625 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 118 2024-11-23T05:42:02,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-23T05:42:02,625 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:42:02,630 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742246_1422 (size=959) 2024-11-23T05:42:02,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-23T05:42:02,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-23T05:42:03,033 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:42:03,040 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742247_1423 (size=53) 2024-11-23T05:42:03,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-23T05:42:03,442 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:42:03,442 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing 96da1e6d03a7b375833ea586fe9dc796, disabling compactions & flushes 2024-11-23T05:42:03,442 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:03,442 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:03,442 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. after waiting 0 ms 2024-11-23T05:42:03,442 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:03,442 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:03,442 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for 96da1e6d03a7b375833ea586fe9dc796: Waiting for close lock at 1732340523442Disabling compacts and flushes for region at 1732340523442Disabling writes for close at 1732340523442Writing region close event to WAL at 1732340523442Closed at 1732340523442 2024-11-23T05:42:03,445 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:42:03,445 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732340523445"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732340523445"}]},"ts":"1732340523445"} 2024-11-23T05:42:03,448 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:42:03,450 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:42:03,451 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340523450"}]},"ts":"1732340523450"} 2024-11-23T05:42:03,453 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T05:42:03,453 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, ASSIGN}] 2024-11-23T05:42:03,454 INFO [PEWorker-2 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, ASSIGN 2024-11-23T05:42:03,455 INFO [PEWorker-2 {}] assignment.TransitRegionStateProcedure(269): Starting pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, ASSIGN; state=OFFLINE, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=false 2024-11-23T05:42:03,606 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=96da1e6d03a7b375833ea586fe9dc796, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:03,609 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=119, ppid=118, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, ASSIGN because future has completed 2024-11-23T05:42:03,609 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:42:03,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(153): Removing adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T05:42:03,767 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-23T05:42:03,770 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:03,770 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7752): Opening region: {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:42:03,772 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,772 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:42:03,772 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7794): checking encryption for 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,772 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(7797): checking classloading for 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,774 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,775 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96da1e6d03a7b375833ea586fe9dc796 columnFamilyName A 2024-11-23T05:42:03,776 DEBUG [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:03,776 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] regionserver.HStore(327): Store=96da1e6d03a7b375833ea586fe9dc796/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:03,776 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,778 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96da1e6d03a7b375833ea586fe9dc796 columnFamilyName B 2024-11-23T05:42:03,778 DEBUG [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:03,779 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] regionserver.HStore(327): Store=96da1e6d03a7b375833ea586fe9dc796/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:03,779 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,781 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region 96da1e6d03a7b375833ea586fe9dc796 columnFamilyName C 2024-11-23T05:42:03,781 DEBUG [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:03,781 INFO [StoreOpener-96da1e6d03a7b375833ea586fe9dc796-1 {}] regionserver.HStore(327): Store=96da1e6d03a7b375833ea586fe9dc796/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:03,781 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1038): replaying wal for 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,782 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,783 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,784 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1048): stopping wal replay for 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,784 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1060): Cleaning up temporary data for 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,785 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:42:03,786 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1093): writing seq id for 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,788 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:42:03,789 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1114): Opened 96da1e6d03a7b375833ea586fe9dc796; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=63304770, jitterRate=-0.056685417890548706}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:42:03,789 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1122): Running coprocessor post-open hooks for 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:03,790 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegion(1006): Region open journal for 96da1e6d03a7b375833ea586fe9dc796: Running coprocessor pre-open hook at 1732340523772Writing region info on filesystem at 1732340523772Initializing all the Stores at 1732340523773 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340523773Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340523774 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340523774Cleaning up temporary data from old regions at 1732340523784 (+10 ms)Running coprocessor post-open hooks at 1732340523789 (+5 ms)Region opened successfully at 1732340523790 (+1 ms) 2024-11-23T05:42:03,790 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., pid=120, masterSystemTime=1732340523762 2024-11-23T05:42:03,792 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:03,792 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=120}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:03,793 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=119 updating hbase:meta row=96da1e6d03a7b375833ea586fe9dc796, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:03,794 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=120, ppid=119, state=RUNNABLE, hasLock=false; OpenRegionProcedure 96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:42:03,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=120, resume processing ppid=119 2024-11-23T05:42:03,797 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=120, ppid=119, state=SUCCESS, hasLock=false; OpenRegionProcedure 96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 in 186 msec 2024-11-23T05:42:03,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=119, resume processing ppid=118 2024-11-23T05:42:03,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=119, ppid=118, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, ASSIGN in 344 msec 2024-11-23T05:42:03,799 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:42:03,800 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340523799"}]},"ts":"1732340523799"} 2024-11-23T05:42:03,801 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T05:42:03,802 INFO [PEWorker-5 {}] procedure.CreateTableProcedure(89): pid=118, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:42:03,803 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=118, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1800 sec 2024-11-23T05:42:04,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=118 2024-11-23T05:42:04,776 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:04,777 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fd12bf6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,777 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,777 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,778 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7cd76b5a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,778 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,779 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,780 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33936, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,780 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@f1c8566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,780 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,781 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,782 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47680, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,783 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:42:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,40189,1732340393249 2024-11-23T05:42:04,783 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@c5131a8 2024-11-23T05:42:04,784 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:42:04,784 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33940, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:42:04,785 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136bd6b8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,785 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,785 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,786 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@19e26d3c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,786 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,787 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,787 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33954, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,788 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6209a436, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,788 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3128f70a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,788 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,788 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,789 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52f4380c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,789 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,790 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33980, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,790 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@407d3929, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,791 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4499c466, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,791 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,791 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,791 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,791 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@18c68bf2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,792 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,792 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34000, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,793 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@52591ed7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,793 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@340f50c7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,793 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,793 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,794 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@72087615, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,794 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,795 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34020, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,795 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@17237ffe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,795 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@34ff7534, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,796 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,796 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,796 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3005ac9, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,796 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,797 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34036, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,797 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2295edb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,798 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64cc712e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,798 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,798 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,799 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5d99c18e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,799 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,800 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,800 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34050, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,801 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@136aa1c1, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,801 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6196a59c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,801 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,801 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,802 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,802 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467039d0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,803 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,804 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34080, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,804 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5742b9b2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,805 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fcc4da7, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,805 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,805 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,805 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,805 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@467ab072, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,806 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,807 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34098, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,807 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@ae61df0, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,808 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@77291cb6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,808 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,808 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,808 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,808 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@362492e6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,809 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,810 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34122, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,810 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@24562d39, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,810 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@541036fe, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,810 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:04,811 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:04,811 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2c961c12, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,811 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:04,812 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:04,812 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:34144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:04,813 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@746b7473, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:04,813 DEBUG [Thread-1914 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,813 DEBUG [Thread-1931 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,813 DEBUG [Thread-1922 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,813 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:04,814 DEBUG [Thread-1927 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,814 DEBUG [Thread-1920 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,814 DEBUG [Thread-1933 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,814 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,815 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees 2024-11-23T05:42:04,815 DEBUG [Thread-1918 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,815 DEBUG [Thread-1931 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,815 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,816 DEBUG [Thread-1916 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,816 DEBUG [Thread-1925 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,816 DEBUG [Thread-1931 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,816 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47690, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,816 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47716, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,817 DEBUG [Thread-1929 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47688, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,817 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47724, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,817 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,818 DEBUG [Thread-1929 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47686, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,818 DEBUG [Thread-1929 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47740, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,818 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47746, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,818 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47754, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:04,819 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47764, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,819 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,820 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:04,820 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,820 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:47774, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,821 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=121, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:04,821 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=122, ppid=121, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:04,821 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,821 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T05:42:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,822 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='', locateType=CURRENT is [region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2] 2024-11-23T05:42:04,827 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:04,827 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:04,827 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:42:04,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/17676d03070a47c69e7e854e04c270e4 is 50, key is test_row_0/A:col10/1732340524826/Put/seqid=0 2024-11-23T05:42:04,850 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742248_1424 (size=14341) 2024-11-23T05:42:04,850 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/17676d03070a47c69e7e854e04c270e4 2024-11-23T05:42:04,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340584863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,871 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340584864, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,873 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,873 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340584866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,874 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,875 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/11240e19f19040fcba2696acdab7c26d is 50, key is test_row_0/B:col10/1732340524826/Put/seqid=0 2024-11-23T05:42:04,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 6 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340584867, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,877 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,877 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,877 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340584868, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,878 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,879 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742249_1425 (size=12001) 2024-11-23T05:42:04,926 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T05:42:04,974 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T05:42:04,974 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:04,975 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:04,975 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:04,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:04,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,984 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340584979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,984 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340584979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,985 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,985 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340584979, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,985 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340584986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,989 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:04,989 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340584986, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:04,989 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,133 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T05:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,133 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,134 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,136 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T05:42:05,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340585196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,201 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,201 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340585196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340585196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340585196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,202 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,202 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,202 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340585197, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,280 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/11240e19f19040fcba2696acdab7c26d 2024-11-23T05:42:05,286 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T05:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,286 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,286 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,302 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e0753e6f5e714c9bbc3e9708f1e19096 is 50, key is test_row_0/C:col10/1732340524826/Put/seqid=0 2024-11-23T05:42:05,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742250_1426 (size=12001) 2024-11-23T05:42:05,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T05:42:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,446 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T05:42:05,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,507 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340585506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,507 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340585506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340585506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,508 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,508 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340585506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,508 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,511 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:05,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340585506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:05,511 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:05,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T05:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] handler.RSProcedureHandler(58): pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=122 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=122 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:05,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=13 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e0753e6f5e714c9bbc3e9708f1e19096 2024-11-23T05:42:05,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/17676d03070a47c69e7e854e04c270e4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/17676d03070a47c69e7e854e04c270e4 2024-11-23T05:42:05,722 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/17676d03070a47c69e7e854e04c270e4, entries=200, sequenceid=13, filesize=14.0 K 2024-11-23T05:42:05,723 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/11240e19f19040fcba2696acdab7c26d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/11240e19f19040fcba2696acdab7c26d 2024-11-23T05:42:05,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/11240e19f19040fcba2696acdab7c26d, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T05:42:05,727 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e0753e6f5e714c9bbc3e9708f1e19096 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e0753e6f5e714c9bbc3e9708f1e19096 2024-11-23T05:42:05,732 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e0753e6f5e714c9bbc3e9708f1e19096, entries=150, sequenceid=13, filesize=11.7 K 2024-11-23T05:42:05,733 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96da1e6d03a7b375833ea586fe9dc796 in 906ms, sequenceid=13, compaction requested=false 2024-11-23T05:42:05,733 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T05:42:05,734 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:05,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=122 2024-11-23T05:42:05,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:05,746 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:42:05,750 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/4651eceddf384d2f9a85805735110ea5 is 50, key is test_row_0/A:col10/1732340524862/Put/seqid=0 2024-11-23T05:42:05,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742251_1427 (size=12001) 2024-11-23T05:42:05,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T05:42:06,021 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:06,021 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:06,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340586024, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,032 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,032 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340586025, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,032 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,033 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,033 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340586031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,034 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,034 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340586031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,034 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,043 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340586031, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,144 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340586136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,144 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,144 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340586136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340586136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,145 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340586136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,145 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,147 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,147 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340586146, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,147 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,148 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,175 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/4651eceddf384d2f9a85805735110ea5 2024-11-23T05:42:06,192 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/25ab75c8f9f441e39f39d1df19ff92b4 is 50, key is test_row_0/B:col10/1732340524862/Put/seqid=0 2024-11-23T05:42:06,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742252_1428 (size=12001) 2024-11-23T05:42:06,214 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/25ab75c8f9f441e39f39d1df19ff92b4 2024-11-23T05:42:06,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/cd86a9ab39214fcd810b030f998fd77f is 50, key is test_row_0/C:col10/1732340524862/Put/seqid=0 2024-11-23T05:42:06,226 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742253_1429 (size=12001) 2024-11-23T05:42:06,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,363 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340586356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,363 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,363 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340586356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340586356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340586357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,364 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,364 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340586357, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,364 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,466 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:42:06,627 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=37 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/cd86a9ab39214fcd810b030f998fd77f 2024-11-23T05:42:06,632 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/4651eceddf384d2f9a85805735110ea5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/4651eceddf384d2f9a85805735110ea5 2024-11-23T05:42:06,639 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/4651eceddf384d2f9a85805735110ea5, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T05:42:06,640 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/25ab75c8f9f441e39f39d1df19ff92b4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/25ab75c8f9f441e39f39d1df19ff92b4 2024-11-23T05:42:06,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/25ab75c8f9f441e39f39d1df19ff92b4, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T05:42:06,643 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/cd86a9ab39214fcd810b030f998fd77f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cd86a9ab39214fcd810b030f998fd77f 2024-11-23T05:42:06,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cd86a9ab39214fcd810b030f998fd77f, entries=150, sequenceid=37, filesize=11.7 K 2024-11-23T05:42:06,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 96da1e6d03a7b375833ea586fe9dc796 in 903ms, sequenceid=37, compaction requested=false 2024-11-23T05:42:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:06,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=122}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=122 2024-11-23T05:42:06,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=122 2024-11-23T05:42:06,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=122, resume processing ppid=121 2024-11-23T05:42:06,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=122, ppid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8290 sec 2024-11-23T05:42:06,655 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=121, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=121, table=TestAcidGuarantees in 1.8410 sec 2024-11-23T05:42:06,670 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:06,670 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:06,671 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:42:06,680 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/c922bb5d7cbd422b9d983d07ceffb0b2 is 50, key is test_row_0/A:col10/1732340526029/Put/seqid=0 2024-11-23T05:42:06,685 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742254_1430 (size=14341) 2024-11-23T05:42:06,687 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/c922bb5d7cbd422b9d983d07ceffb0b2 2024-11-23T05:42:06,692 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/3604d796da2c40249836dce287aa383e is 50, key is test_row_0/B:col10/1732340526029/Put/seqid=0 2024-11-23T05:42:06,696 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742255_1431 (size=12001) 2024-11-23T05:42:06,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340586707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,726 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340586712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340586713, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,727 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340586714, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,728 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,728 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340586715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,728 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,729 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340586825, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,829 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,841 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,841 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340586836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340586836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340586836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,842 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,842 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:06,842 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340586836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:06,843 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:06,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=121 2024-11-23T05:42:06,966 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:06,967 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees 2024-11-23T05:42:06,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T05:42:06,968 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:06,969 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=123, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:06,969 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=124, ppid=123, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:07,042 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340587036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,043 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,052 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340587046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340587046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340587046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,053 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,053 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,054 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340587046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,054 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T05:42:07,097 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/3604d796da2c40249836dce287aa383e 2024-11-23T05:42:07,103 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e125235eef34466f92f90544b46f5f18 is 50, key is test_row_0/C:col10/1732340526029/Put/seqid=0 2024-11-23T05:42:07,109 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742256_1432 (size=12001) 2024-11-23T05:42:07,110 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=50 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e125235eef34466f92f90544b46f5f18 2024-11-23T05:42:07,114 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/c922bb5d7cbd422b9d983d07ceffb0b2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/c922bb5d7cbd422b9d983d07ceffb0b2 2024-11-23T05:42:07,117 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/c922bb5d7cbd422b9d983d07ceffb0b2, entries=200, sequenceid=50, filesize=14.0 K 2024-11-23T05:42:07,121 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T05:42:07,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:07,121 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:07,122 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] handler.RSProcedureHandler(58): pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:07,122 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=124 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:07,122 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=124 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:07,130 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/3604d796da2c40249836dce287aa383e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3604d796da2c40249836dce287aa383e 2024-11-23T05:42:07,133 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3604d796da2c40249836dce287aa383e, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T05:42:07,134 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e125235eef34466f92f90544b46f5f18 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e125235eef34466f92f90544b46f5f18 2024-11-23T05:42:07,139 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e125235eef34466f92f90544b46f5f18, entries=150, sequenceid=50, filesize=11.7 K 2024-11-23T05:42:07,140 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for 96da1e6d03a7b375833ea586fe9dc796 in 470ms, sequenceid=50, compaction requested=true 2024-11-23T05:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:07,140 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:07,140 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:07,140 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:07,141 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:07,141 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:07,141 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:07,141 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/11240e19f19040fcba2696acdab7c26d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/25ab75c8f9f441e39f39d1df19ff92b4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3604d796da2c40249836dce287aa383e] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=35.2 K 2024-11-23T05:42:07,141 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 11240e19f19040fcba2696acdab7c26d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732340524825 2024-11-23T05:42:07,142 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 40683 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:07,142 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:07,142 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:07,142 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/17676d03070a47c69e7e854e04c270e4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/4651eceddf384d2f9a85805735110ea5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/c922bb5d7cbd422b9d983d07ceffb0b2] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=39.7 K 2024-11-23T05:42:07,142 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 25ab75c8f9f441e39f39d1df19ff92b4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732340524862 2024-11-23T05:42:07,143 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 17676d03070a47c69e7e854e04c270e4, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732340524824 2024-11-23T05:42:07,143 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3604d796da2c40249836dce287aa383e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340526029 2024-11-23T05:42:07,143 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4651eceddf384d2f9a85805735110ea5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732340524862 2024-11-23T05:42:07,143 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting c922bb5d7cbd422b9d983d07ceffb0b2, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340526029 2024-11-23T05:42:07,160 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#359 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:07,161 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/e895ace984b94e23aaa49848cbe46eec is 50, key is test_row_0/A:col10/1732340526029/Put/seqid=0 2024-11-23T05:42:07,162 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#360 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:07,162 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/10a55d34308d4395b122a18725bfdca2 is 50, key is test_row_0/B:col10/1732340526029/Put/seqid=0 2024-11-23T05:42:07,180 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742258_1434 (size=12104) 2024-11-23T05:42:07,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742257_1433 (size=12104) 2024-11-23T05:42:07,191 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/e895ace984b94e23aaa49848cbe46eec as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e895ace984b94e23aaa49848cbe46eec 2024-11-23T05:42:07,195 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into e895ace984b94e23aaa49848cbe46eec(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:07,195 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:07,195 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=13, startTime=1732340527140; duration=0sec 2024-11-23T05:42:07,195 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:07,195 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:07,195 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:07,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:07,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:07,198 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:07,198 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e0753e6f5e714c9bbc3e9708f1e19096, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cd86a9ab39214fcd810b030f998fd77f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e125235eef34466f92f90544b46f5f18] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=35.2 K 2024-11-23T05:42:07,198 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e0753e6f5e714c9bbc3e9708f1e19096, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=13, earliestPutTs=1732340524825 2024-11-23T05:42:07,199 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting cd86a9ab39214fcd810b030f998fd77f, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=37, earliestPutTs=1732340524862 2024-11-23T05:42:07,199 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e125235eef34466f92f90544b46f5f18, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340526029 2024-11-23T05:42:07,204 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#361 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:07,205 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/382dce48b7ce45fface8da443d7e4f9f is 50, key is test_row_0/C:col10/1732340526029/Put/seqid=0 2024-11-23T05:42:07,213 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742259_1435 (size=12104) 2024-11-23T05:42:07,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=124 2024-11-23T05:42:07,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:07,274 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:42:07,278 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/e41dee3c045c46b98277ba03af073eeb is 50, key is test_row_0/A:col10/1732340526714/Put/seqid=0 2024-11-23T05:42:07,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T05:42:07,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742260_1436 (size=12001) 2024-11-23T05:42:07,351 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:07,351 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340587380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,395 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340587380, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,395 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340587381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,396 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,396 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340587382, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340587381, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,396 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,397 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340587506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,510 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340587506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340587506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340587506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,511 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,511 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,511 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340587506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,512 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,584 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/10a55d34308d4395b122a18725bfdca2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/10a55d34308d4395b122a18725bfdca2 2024-11-23T05:42:07,588 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into 10a55d34308d4395b122a18725bfdca2(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:07,588 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:07,588 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=13, startTime=1732340527140; duration=0sec 2024-11-23T05:42:07,588 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:07,588 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:07,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T05:42:07,617 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/382dce48b7ce45fface8da443d7e4f9f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/382dce48b7ce45fface8da443d7e4f9f 2024-11-23T05:42:07,621 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into 382dce48b7ce45fface8da443d7e4f9f(size=11.8 K), total size for store is 11.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:07,621 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:07,621 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=13, startTime=1732340527140; duration=0sec 2024-11-23T05:42:07,621 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:07,621 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:07,688 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/e41dee3c045c46b98277ba03af073eeb 2024-11-23T05:42:07,694 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/99e69acf48ae475084f02f3d8bd77549 is 50, key is test_row_0/B:col10/1732340526714/Put/seqid=0 2024-11-23T05:42:07,698 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742261_1437 (size=12001) 2024-11-23T05:42:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,722 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340587716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,722 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,722 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,723 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340587716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,723 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340587716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,725 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,725 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,725 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 30 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340587716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:07,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340587716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:07,726 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,031 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340588026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,031 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340588036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340588036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,041 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,041 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340588036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,042 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,042 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340588036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,042 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,098 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/99e69acf48ae475084f02f3d8bd77549 2024-11-23T05:42:08,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T05:42:08,107 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d0495ca3bc734ffbbf5ffbc678b16bc1 is 50, key is test_row_0/C:col10/1732340526714/Put/seqid=0 2024-11-23T05:42:08,112 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742262_1438 (size=12001) 2024-11-23T05:42:08,512 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=74 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d0495ca3bc734ffbbf5ffbc678b16bc1 2024-11-23T05:42:08,516 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/e41dee3c045c46b98277ba03af073eeb as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e41dee3c045c46b98277ba03af073eeb 2024-11-23T05:42:08,527 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e41dee3c045c46b98277ba03af073eeb, entries=150, sequenceid=74, filesize=11.7 K 2024-11-23T05:42:08,528 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/99e69acf48ae475084f02f3d8bd77549 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/99e69acf48ae475084f02f3d8bd77549 2024-11-23T05:42:08,532 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/99e69acf48ae475084f02f3d8bd77549, entries=150, sequenceid=74, filesize=11.7 K 2024-11-23T05:42:08,533 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d0495ca3bc734ffbbf5ffbc678b16bc1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d0495ca3bc734ffbbf5ffbc678b16bc1 2024-11-23T05:42:08,536 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d0495ca3bc734ffbbf5ffbc678b16bc1, entries=150, sequenceid=74, filesize=11.7 K 2024-11-23T05:42:08,537 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for 96da1e6d03a7b375833ea586fe9dc796 in 1264ms, sequenceid=74, compaction requested=false 2024-11-23T05:42:08,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:08,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:08,537 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=124}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=124 2024-11-23T05:42:08,538 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=124 2024-11-23T05:42:08,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=124, resume processing ppid=123 2024-11-23T05:42:08,540 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=124, ppid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.5690 sec 2024-11-23T05:42:08,540 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:08,541 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:08,541 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:42:08,541 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=123, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=123, table=TestAcidGuarantees in 1.5730 sec 2024-11-23T05:42:08,557 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/1f1e158b07094b7bbf41a9d60aa494fc is 50, key is test_row_0/A:col10/1732340527379/Put/seqid=0 2024-11-23T05:42:08,561 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742263_1439 (size=14341) 2024-11-23T05:42:08,562 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/1f1e158b07094b7bbf41a9d60aa494fc 2024-11-23T05:42:08,570 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/96552792406746f69a96bb974e5560ba is 50, key is test_row_0/B:col10/1732340527379/Put/seqid=0 2024-11-23T05:42:08,572 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742264_1440 (size=12001) 2024-11-23T05:42:08,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,598 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340588586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,598 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340588586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,599 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340588592, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340588597, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,604 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,605 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,610 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340588598, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,610 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,713 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,713 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340588707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,713 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,714 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340588707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,714 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,715 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,715 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340588707, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,715 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,716 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,728 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340588716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,729 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340588716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,729 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,934 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,934 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340588926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340588926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,935 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340588926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,936 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,945 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340588936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,946 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340588936, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:08,946 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:08,973 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/96552792406746f69a96bb974e5560ba 2024-11-23T05:42:08,980 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d037aaf5c1024de89e5c8bdc414fe58d is 50, key is test_row_0/C:col10/1732340527379/Put/seqid=0 2024-11-23T05:42:08,992 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742265_1441 (size=12001) 2024-11-23T05:42:08,992 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d037aaf5c1024de89e5c8bdc414fe58d 2024-11-23T05:42:08,998 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/1f1e158b07094b7bbf41a9d60aa494fc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/1f1e158b07094b7bbf41a9d60aa494fc 2024-11-23T05:42:09,001 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/1f1e158b07094b7bbf41a9d60aa494fc, entries=200, sequenceid=90, filesize=14.0 K 2024-11-23T05:42:09,002 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/96552792406746f69a96bb974e5560ba as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/96552792406746f69a96bb974e5560ba 2024-11-23T05:42:09,009 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/96552792406746f69a96bb974e5560ba, entries=150, sequenceid=90, filesize=11.7 K 2024-11-23T05:42:09,011 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d037aaf5c1024de89e5c8bdc414fe58d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d037aaf5c1024de89e5c8bdc414fe58d 2024-11-23T05:42:09,022 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d037aaf5c1024de89e5c8bdc414fe58d, entries=150, sequenceid=90, filesize=11.7 K 2024-11-23T05:42:09,023 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for 96da1e6d03a7b375833ea586fe9dc796 in 482ms, sequenceid=90, compaction requested=true 2024-11-23T05:42:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:09,023 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:09,023 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:09,024 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T05:42:09,024 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:09,024 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 38446 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:09,025 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:09,025 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,025 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e895ace984b94e23aaa49848cbe46eec, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e41dee3c045c46b98277ba03af073eeb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/1f1e158b07094b7bbf41a9d60aa494fc] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=37.5 K 2024-11-23T05:42:09,025 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:09,025 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:09,025 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,025 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/10a55d34308d4395b122a18725bfdca2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/99e69acf48ae475084f02f3d8bd77549, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/96552792406746f69a96bb974e5560ba] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=35.3 K 2024-11-23T05:42:09,027 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e895ace984b94e23aaa49848cbe46eec, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340526029 2024-11-23T05:42:09,028 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 10a55d34308d4395b122a18725bfdca2, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340526029 2024-11-23T05:42:09,028 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e41dee3c045c46b98277ba03af073eeb, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732340526705 2024-11-23T05:42:09,028 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 1f1e158b07094b7bbf41a9d60aa494fc, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732340527379 2024-11-23T05:42:09,028 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 99e69acf48ae475084f02f3d8bd77549, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732340526705 2024-11-23T05:42:09,029 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 96552792406746f69a96bb974e5560ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732340527379 2024-11-23T05:42:09,035 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#368 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:09,035 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/4b2dcbe18dd44af3af55b16bd1435222 is 50, key is test_row_0/B:col10/1732340527379/Put/seqid=0 2024-11-23T05:42:09,049 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#369 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:09,050 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/20ef81033950492db97c13cddd2c85db is 50, key is test_row_0/A:col10/1732340527379/Put/seqid=0 2024-11-23T05:42:09,084 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742266_1442 (size=12207) 2024-11-23T05:42:09,090 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/4b2dcbe18dd44af3af55b16bd1435222 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/4b2dcbe18dd44af3af55b16bd1435222 2024-11-23T05:42:09,097 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into 4b2dcbe18dd44af3af55b16bd1435222(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:09,097 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:09,097 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=13, startTime=1732340529023; duration=0sec 2024-11-23T05:42:09,097 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:09,097 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:09,097 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:09,100 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36106 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:09,100 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:09,100 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,101 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/382dce48b7ce45fface8da443d7e4f9f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d0495ca3bc734ffbbf5ffbc678b16bc1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d037aaf5c1024de89e5c8bdc414fe58d] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=35.3 K 2024-11-23T05:42:09,102 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 382dce48b7ce45fface8da443d7e4f9f, keycount=150, bloomtype=ROW, size=11.8 K, encoding=NONE, compression=NONE, seqNum=50, earliestPutTs=1732340526029 2024-11-23T05:42:09,102 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d0495ca3bc734ffbbf5ffbc678b16bc1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=74, earliestPutTs=1732340526705 2024-11-23T05:42:09,103 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d037aaf5c1024de89e5c8bdc414fe58d, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732340527379 2024-11-23T05:42:09,108 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742267_1443 (size=12207) 2024-11-23T05:42:09,112 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#370 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:09,113 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/338cc5bc57814f1ba91edad5ed370c48 is 50, key is test_row_0/C:col10/1732340527379/Put/seqid=0 2024-11-23T05:42:09,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=123 2024-11-23T05:42:09,117 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:09,117 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:09,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees 2024-11-23T05:42:09,118 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:09,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T05:42:09,119 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=125, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:09,119 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=126, ppid=125, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:09,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742268_1444 (size=12207) 2024-11-23T05:42:09,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T05:42:09,254 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:09,254 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:09,254 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:42:09,261 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/045fbff69ee64516982dfa74ae67e564 is 50, key is test_row_0/A:col10/1732340529253/Put/seqid=0 2024-11-23T05:42:09,268 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742269_1445 (size=14341) 2024-11-23T05:42:09,271 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T05:42:09,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:09,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,272 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,278 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340589267, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340589268, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,279 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,279 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340589277, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,280 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,288 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340589279, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,289 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,291 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,291 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340589280, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,292 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340589386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,391 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340589386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,391 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,392 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,392 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340589386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,392 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,403 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340589396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,404 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,404 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340589396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,404 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T05:42:09,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:09,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T05:42:09,511 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/20ef81033950492db97c13cddd2c85db as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/20ef81033950492db97c13cddd2c85db 2024-11-23T05:42:09,517 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into 20ef81033950492db97c13cddd2c85db(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:09,517 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:09,517 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=13, startTime=1732340529023; duration=0sec 2024-11-23T05:42:09,517 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:09,517 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:09,534 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/338cc5bc57814f1ba91edad5ed370c48 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/338cc5bc57814f1ba91edad5ed370c48 2024-11-23T05:42:09,544 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into 338cc5bc57814f1ba91edad5ed370c48(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:09,544 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:09,544 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=13, startTime=1732340529023; duration=0sec 2024-11-23T05:42:09,544 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:09,544 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:09,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T05:42:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:09,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,577 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340589596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,603 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340589596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,604 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,604 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340589596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,604 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340589616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,621 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,621 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,621 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340589616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,622 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,668 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/045fbff69ee64516982dfa74ae67e564 2024-11-23T05:42:09,674 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/45fd6f624a7942978c7c02a0e2bb135e is 50, key is test_row_0/B:col10/1732340529253/Put/seqid=0 2024-11-23T05:42:09,683 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742270_1446 (size=12001) 2024-11-23T05:42:09,729 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T05:42:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:09,729 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,729 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,730 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T05:42:09,885 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T05:42:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:09,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:09,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:09,914 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340589906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,915 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,915 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340589906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,915 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,921 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,921 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340589916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,923 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340589926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,933 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,933 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340589926, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:09,933 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:09,934 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:09,935 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,038 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T05:42:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:10,038 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:10,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] handler.RSProcedureHandler(58): pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:10,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=126 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:10,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=126 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:10,083 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/45fd6f624a7942978c7c02a0e2bb135e 2024-11-23T05:42:10,088 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/6a684aa0f8a94deea669c42d2dde0022 is 50, key is test_row_0/C:col10/1732340529253/Put/seqid=0 2024-11-23T05:42:10,091 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742271_1447 (size=12001) 2024-11-23T05:42:10,092 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=114 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/6a684aa0f8a94deea669c42d2dde0022 2024-11-23T05:42:10,094 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/045fbff69ee64516982dfa74ae67e564 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/045fbff69ee64516982dfa74ae67e564 2024-11-23T05:42:10,099 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/045fbff69ee64516982dfa74ae67e564, entries=200, sequenceid=114, filesize=14.0 K 2024-11-23T05:42:10,099 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/45fd6f624a7942978c7c02a0e2bb135e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/45fd6f624a7942978c7c02a0e2bb135e 2024-11-23T05:42:10,102 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/45fd6f624a7942978c7c02a0e2bb135e, entries=150, sequenceid=114, filesize=11.7 K 2024-11-23T05:42:10,102 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/6a684aa0f8a94deea669c42d2dde0022 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/6a684aa0f8a94deea669c42d2dde0022 2024-11-23T05:42:10,107 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/6a684aa0f8a94deea669c42d2dde0022, entries=150, sequenceid=114, filesize=11.7 K 2024-11-23T05:42:10,107 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=73.80 KB/75570 for 96da1e6d03a7b375833ea586fe9dc796 in 853ms, sequenceid=114, compaction requested=false 2024-11-23T05:42:10,108 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:10,191 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=126 2024-11-23T05:42:10,191 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:10,192 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:42:10,196 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/50be77dcf9484badaf1693fbef2b112c is 50, key is test_row_0/A:col10/1732340529276/Put/seqid=0 2024-11-23T05:42:10,211 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742272_1448 (size=12001) 2024-11-23T05:42:10,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T05:42:10,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:10,429 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340590473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,475 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,475 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340590473, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,475 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,476 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,485 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,485 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340590474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,486 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340590474, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,486 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,486 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,486 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340590475, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,487 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,592 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340590586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,592 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,592 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,593 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 47 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340590586, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,593 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,602 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,602 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340590596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340590596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,603 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,603 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340590596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,603 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,612 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/50be77dcf9484badaf1693fbef2b112c 2024-11-23T05:42:10,618 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/66dbcc6ef88c4ae393457fd827f35fe0 is 50, key is test_row_0/B:col10/1732340529276/Put/seqid=0 2024-11-23T05:42:10,621 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742273_1449 (size=12001) 2024-11-23T05:42:10,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340590796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340590796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,802 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,811 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340590806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340590806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:10,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340590806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:10,812 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,022 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/66dbcc6ef88c4ae393457fd827f35fe0 2024-11-23T05:42:11,027 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d4ae200dbbab46e59f1c16e4a7ee66fd is 50, key is test_row_0/C:col10/1732340529276/Put/seqid=0 2024-11-23T05:42:11,033 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742274_1450 (size=12001) 2024-11-23T05:42:11,111 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,111 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340591106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,112 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,112 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340591106, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,112 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340591116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,123 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,123 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,123 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340591116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,124 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,124 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340591116, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,124 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,125 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,265 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T05:42:11,434 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=130 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d4ae200dbbab46e59f1c16e4a7ee66fd 2024-11-23T05:42:11,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/50be77dcf9484badaf1693fbef2b112c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/50be77dcf9484badaf1693fbef2b112c 2024-11-23T05:42:11,440 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/50be77dcf9484badaf1693fbef2b112c, entries=150, sequenceid=130, filesize=11.7 K 2024-11-23T05:42:11,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/66dbcc6ef88c4ae393457fd827f35fe0 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/66dbcc6ef88c4ae393457fd827f35fe0 2024-11-23T05:42:11,443 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/66dbcc6ef88c4ae393457fd827f35fe0, entries=150, sequenceid=130, filesize=11.7 K 2024-11-23T05:42:11,443 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d4ae200dbbab46e59f1c16e4a7ee66fd as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4ae200dbbab46e59f1c16e4a7ee66fd 2024-11-23T05:42:11,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4ae200dbbab46e59f1c16e4a7ee66fd, entries=150, sequenceid=130, filesize=11.7 K 2024-11-23T05:42:11,447 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=134.18 KB/137400 for 96da1e6d03a7b375833ea586fe9dc796 in 1256ms, sequenceid=130, compaction requested=true 2024-11-23T05:42:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:11,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=126}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=126 2024-11-23T05:42:11,447 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=126 2024-11-23T05:42:11,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=126, resume processing ppid=125 2024-11-23T05:42:11,449 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=126, ppid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3290 sec 2024-11-23T05:42:11,450 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=125, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=125, table=TestAcidGuarantees in 2.3330 sec 2024-11-23T05:42:11,619 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:11,619 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:11,619 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:42:11,623 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/3327e155e05342aaae9b1748e1c51258 is 50, key is test_row_0/A:col10/1732340530474/Put/seqid=0 2024-11-23T05:42:11,627 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742275_1451 (size=14541) 2024-11-23T05:42:11,645 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340591635, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,646 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340591636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,646 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340591644, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340591646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,655 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,655 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340591646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,655 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,656 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340591766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340591766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,771 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,771 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340591766, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,771 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,772 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340591975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340591976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,981 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:11,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340591976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:11,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,028 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/3327e155e05342aaae9b1748e1c51258 2024-11-23T05:42:12,035 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6c74a1c15116456e918b35339ea2572f is 50, key is test_row_0/B:col10/1732340530474/Put/seqid=0 2024-11-23T05:42:12,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742276_1452 (size=12151) 2024-11-23T05:42:12,293 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340592286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340592286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,294 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,294 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340592286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,294 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,455 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6c74a1c15116456e918b35339ea2572f 2024-11-23T05:42:12,460 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/ab833def08394ca699184ab7364f5bef is 50, key is test_row_0/C:col10/1732340530474/Put/seqid=0 2024-11-23T05:42:12,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742277_1453 (size=12151) 2024-11-23T05:42:12,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340592656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,667 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,667 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340592656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,667 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:12,668 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2193 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,802 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,802 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340592796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,802 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340592806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,812 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 58 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340592806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:12,812 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:12,864 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=154 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/ab833def08394ca699184ab7364f5bef 2024-11-23T05:42:12,867 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/3327e155e05342aaae9b1748e1c51258 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/3327e155e05342aaae9b1748e1c51258 2024-11-23T05:42:12,870 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/3327e155e05342aaae9b1748e1c51258, entries=200, sequenceid=154, filesize=14.2 K 2024-11-23T05:42:12,870 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6c74a1c15116456e918b35339ea2572f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6c74a1c15116456e918b35339ea2572f 2024-11-23T05:42:12,873 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6c74a1c15116456e918b35339ea2572f, entries=150, sequenceid=154, filesize=11.9 K 2024-11-23T05:42:12,874 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/ab833def08394ca699184ab7364f5bef as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/ab833def08394ca699184ab7364f5bef 2024-11-23T05:42:12,877 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/ab833def08394ca699184ab7364f5bef, entries=150, sequenceid=154, filesize=11.9 K 2024-11-23T05:42:12,877 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for 96da1e6d03a7b375833ea586fe9dc796 in 1258ms, sequenceid=154, compaction requested=true 2024-11-23T05:42:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:12,878 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:12,878 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=2), splitQueue=0 2024-11-23T05:42:12,878 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:12,879 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 53090 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:12,879 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:12,879 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:12,879 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/20ef81033950492db97c13cddd2c85db, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/045fbff69ee64516982dfa74ae67e564, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/50be77dcf9484badaf1693fbef2b112c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/3327e155e05342aaae9b1748e1c51258] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=51.8 K 2024-11-23T05:42:12,879 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:12,879 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:12,879 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:12,879 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/4b2dcbe18dd44af3af55b16bd1435222, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/45fd6f624a7942978c7c02a0e2bb135e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/66dbcc6ef88c4ae393457fd827f35fe0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6c74a1c15116456e918b35339ea2572f] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=47.2 K 2024-11-23T05:42:12,880 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 20ef81033950492db97c13cddd2c85db, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732340527379 2024-11-23T05:42:12,880 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4b2dcbe18dd44af3af55b16bd1435222, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732340527379 2024-11-23T05:42:12,880 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 045fbff69ee64516982dfa74ae67e564, keycount=200, bloomtype=ROW, size=14.0 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732340528582 2024-11-23T05:42:12,880 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 45fd6f624a7942978c7c02a0e2bb135e, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732340528595 2024-11-23T05:42:12,880 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50be77dcf9484badaf1693fbef2b112c, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732340529267 2024-11-23T05:42:12,880 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3327e155e05342aaae9b1748e1c51258, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732340530471 2024-11-23T05:42:12,881 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 66dbcc6ef88c4ae393457fd827f35fe0, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732340529267 2024-11-23T05:42:12,881 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6c74a1c15116456e918b35339ea2572f, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732340530471 2024-11-23T05:42:12,887 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#380 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:12,888 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/7b67c24fc8cd4610b2bfa15e55f2c2f8 is 50, key is test_row_0/A:col10/1732340530474/Put/seqid=0 2024-11-23T05:42:12,896 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#381 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:12,897 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/69c53878def944279863ef32ca9e6ced is 50, key is test_row_0/B:col10/1732340530474/Put/seqid=0 2024-11-23T05:42:12,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742278_1454 (size=12493) 2024-11-23T05:42:12,902 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/7b67c24fc8cd4610b2bfa15e55f2c2f8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7b67c24fc8cd4610b2bfa15e55f2c2f8 2024-11-23T05:42:12,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742279_1455 (size=12493) 2024-11-23T05:42:12,909 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into 7b67c24fc8cd4610b2bfa15e55f2c2f8(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:12,909 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:12,909 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=12, startTime=1732340532878; duration=0sec 2024-11-23T05:42:12,909 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:12,909 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:12,909 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:12,909 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/69c53878def944279863ef32ca9e6ced as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/69c53878def944279863ef32ca9e6ced 2024-11-23T05:42:12,910 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48360 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:12,910 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:12,910 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:12,910 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/338cc5bc57814f1ba91edad5ed370c48, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/6a684aa0f8a94deea669c42d2dde0022, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4ae200dbbab46e59f1c16e4a7ee66fd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/ab833def08394ca699184ab7364f5bef] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=47.2 K 2024-11-23T05:42:12,910 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 338cc5bc57814f1ba91edad5ed370c48, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=90, earliestPutTs=1732340527379 2024-11-23T05:42:12,910 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 6a684aa0f8a94deea669c42d2dde0022, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=114, earliestPutTs=1732340528595 2024-11-23T05:42:12,911 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4ae200dbbab46e59f1c16e4a7ee66fd, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=130, earliestPutTs=1732340529267 2024-11-23T05:42:12,911 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ab833def08394ca699184ab7364f5bef, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732340530471 2024-11-23T05:42:12,913 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into 69c53878def944279863ef32ca9e6ced(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:12,914 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:12,914 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=12, startTime=1732340532878; duration=0sec 2024-11-23T05:42:12,914 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:12,914 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:12,918 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#382 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:12,918 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/0278dedfe40749fa9c5acbe99210dd27 is 50, key is test_row_0/C:col10/1732340530474/Put/seqid=0 2024-11-23T05:42:12,929 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742280_1456 (size=12493) 2024-11-23T05:42:12,933 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/0278dedfe40749fa9c5acbe99210dd27 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/0278dedfe40749fa9c5acbe99210dd27 2024-11-23T05:42:12,936 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into 0278dedfe40749fa9c5acbe99210dd27(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:12,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:12,936 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=12, startTime=1732340532878; duration=0sec 2024-11-23T05:42:12,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:12,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:13,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=125 2024-11-23T05:42:13,276 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:13,276 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:13,277 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees 2024-11-23T05:42:13,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T05:42:13,278 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:13,278 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=127, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:13,279 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=128, ppid=127, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:13,386 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T05:42:13,430 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=128 2024-11-23T05:42:13,430 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:13,431 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:42:13,434 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/82c7ff5d0e134427bf739d6379afccc3 is 50, key is test_row_0/A:col10/1732340531644/Put/seqid=0 2024-11-23T05:42:13,437 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742281_1457 (size=12151) 2024-11-23T05:42:13,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T05:42:13,697 DEBUG [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(131): Registering adapter for the MetricRegistry: RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees 2024-11-23T05:42:13,697 INFO [HBase-Metrics2-1 {}] impl.GlobalMetricRegistriesAdapter(135): Registering RegionServer,sub=TableRequests_Namespace_default_table_TestAcidGuarantees Metrics about Tables on a single HBase RegionServer 2024-11-23T05:42:13,809 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:13,809 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:13,838 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/82c7ff5d0e134427bf739d6379afccc3 2024-11-23T05:42:13,843 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/1a74ad8cec844bd89feb28f274800e4e is 50, key is test_row_0/B:col10/1732340531644/Put/seqid=0 2024-11-23T05:42:13,851 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742282_1458 (size=12151) 2024-11-23T05:42:13,870 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:13,870 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340593859, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,871 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:13,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340593865, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:13,873 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:13,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340593866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,873 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:13,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T05:42:13,981 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340593975, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:13,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340593976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,982 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:13,982 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340593976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:13,982 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,192 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,192 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340594186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340594186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,193 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,193 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340594186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,193 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,252 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/1a74ad8cec844bd89feb28f274800e4e 2024-11-23T05:42:14,257 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/a23a0287a09e424ca2ae29a743b744d8 is 50, key is test_row_0/C:col10/1732340531644/Put/seqid=0 2024-11-23T05:42:14,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742283_1459 (size=12151) 2024-11-23T05:42:14,416 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T05:42:14,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340594496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,500 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,500 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340594496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,500 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,513 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340594506, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,513 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,662 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=169 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/a23a0287a09e424ca2ae29a743b744d8 2024-11-23T05:42:14,665 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/82c7ff5d0e134427bf739d6379afccc3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/82c7ff5d0e134427bf739d6379afccc3 2024-11-23T05:42:14,668 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/82c7ff5d0e134427bf739d6379afccc3, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T05:42:14,668 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/1a74ad8cec844bd89feb28f274800e4e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/1a74ad8cec844bd89feb28f274800e4e 2024-11-23T05:42:14,671 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/1a74ad8cec844bd89feb28f274800e4e, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T05:42:14,672 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/a23a0287a09e424ca2ae29a743b744d8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a23a0287a09e424ca2ae29a743b744d8 2024-11-23T05:42:14,674 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a23a0287a09e424ca2ae29a743b744d8, entries=150, sequenceid=169, filesize=11.9 K 2024-11-23T05:42:14,675 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96da1e6d03a7b375833ea586fe9dc796 in 1244ms, sequenceid=169, compaction requested=false 2024-11-23T05:42:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:14,675 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=128}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=128 2024-11-23T05:42:14,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=128 2024-11-23T05:42:14,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=128, resume processing ppid=127 2024-11-23T05:42:14,678 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=128, ppid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.3980 sec 2024-11-23T05:42:14,679 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=127, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=127, table=TestAcidGuarantees in 1.4020 sec 2024-11-23T05:42:14,690 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:14,690 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:14,691 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:42:14,695 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/36e94749e13247fab28701c92f6062ff is 50, key is test_row_0/A:col10/1732340533835/Put/seqid=0 2024-11-23T05:42:14,700 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742284_1460 (size=14541) 2024-11-23T05:42:14,717 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,717 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340594712, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,717 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,720 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340594715, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,720 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340594826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,832 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,832 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340594826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:14,832 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,010 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,010 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340595005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,011 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,011 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340595006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,011 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,030 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,030 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340595026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,030 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340595036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,044 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,044 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340595037, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,045 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,046 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,101 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/36e94749e13247fab28701c92f6062ff 2024-11-23T05:42:15,107 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/0f62e1a7dd254283b662a814b8f67e02 is 50, key is test_row_0/B:col10/1732340533835/Put/seqid=0 2024-11-23T05:42:15,111 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742285_1461 (size=12151) 2024-11-23T05:42:15,364 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,365 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,365 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340595356, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,365 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=127 2024-11-23T05:42:15,426 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:15,427 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:15,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees 2024-11-23T05:42:15,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T05:42:15,428 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:15,429 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=129, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:15,429 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=130, ppid=129, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:15,512 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/0f62e1a7dd254283b662a814b8f67e02 2024-11-23T05:42:15,518 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d4b5786e738043bb817218de4a879f64 is 50, key is test_row_0/C:col10/1732340533835/Put/seqid=0 2024-11-23T05:42:15,521 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742286_1462 (size=12151) 2024-11-23T05:42:15,536 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T05:42:15,581 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:15,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:15,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,581 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,581 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,582 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,733 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,734 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,734 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,735 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,745 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T05:42:15,880 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,881 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,881 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340595876, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:15,881 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:15,886 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:15,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:15,886 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,886 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,887 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:15,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=194 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d4b5786e738043bb817218de4a879f64 2024-11-23T05:42:15,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/36e94749e13247fab28701c92f6062ff as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/36e94749e13247fab28701c92f6062ff 2024-11-23T05:42:15,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/36e94749e13247fab28701c92f6062ff, entries=200, sequenceid=194, filesize=14.2 K 2024-11-23T05:42:15,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/0f62e1a7dd254283b662a814b8f67e02 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0f62e1a7dd254283b662a814b8f67e02 2024-11-23T05:42:15,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0f62e1a7dd254283b662a814b8f67e02, entries=150, sequenceid=194, filesize=11.9 K 2024-11-23T05:42:15,930 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/d4b5786e738043bb817218de4a879f64 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4b5786e738043bb817218de4a879f64 2024-11-23T05:42:15,933 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4b5786e738043bb817218de4a879f64, entries=150, sequenceid=194, filesize=11.9 K 2024-11-23T05:42:15,934 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 96da1e6d03a7b375833ea586fe9dc796 in 1244ms, sequenceid=194, compaction requested=true 2024-11-23T05:42:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:15,934 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:15,934 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:15,934 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:15,934 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:15,934 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39185 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:15,934 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:15,934 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:15,935 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,935 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,935 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/69c53878def944279863ef32ca9e6ced, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/1a74ad8cec844bd89feb28f274800e4e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0f62e1a7dd254283b662a814b8f67e02] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=35.9 K 2024-11-23T05:42:15,935 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7b67c24fc8cd4610b2bfa15e55f2c2f8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/82c7ff5d0e134427bf739d6379afccc3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/36e94749e13247fab28701c92f6062ff] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=38.3 K 2024-11-23T05:42:15,935 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 69c53878def944279863ef32ca9e6ced, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732340530471 2024-11-23T05:42:15,935 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7b67c24fc8cd4610b2bfa15e55f2c2f8, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732340530471 2024-11-23T05:42:15,935 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1a74ad8cec844bd89feb28f274800e4e, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732340531643 2024-11-23T05:42:15,935 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 82c7ff5d0e134427bf739d6379afccc3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732340531643 2024-11-23T05:42:15,936 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0f62e1a7dd254283b662a814b8f67e02, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732340533835 2024-11-23T05:42:15,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 36e94749e13247fab28701c92f6062ff, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732340533835 2024-11-23T05:42:15,944 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#389 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:15,945 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6eb1a433e93446eab19ca15dfc1559f8 is 50, key is test_row_0/B:col10/1732340533835/Put/seqid=0 2024-11-23T05:42:15,951 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#390 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:15,952 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/74b78433e0234547ba1ae7634f2139a8 is 50, key is test_row_0/A:col10/1732340533835/Put/seqid=0 2024-11-23T05:42:15,953 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742287_1463 (size=12595) 2024-11-23T05:42:15,960 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742288_1464 (size=12595) 2024-11-23T05:42:15,960 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6eb1a433e93446eab19ca15dfc1559f8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6eb1a433e93446eab19ca15dfc1559f8 2024-11-23T05:42:15,964 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/74b78433e0234547ba1ae7634f2139a8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/74b78433e0234547ba1ae7634f2139a8 2024-11-23T05:42:15,966 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into 6eb1a433e93446eab19ca15dfc1559f8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:15,966 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:15,966 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=13, startTime=1732340535934; duration=0sec 2024-11-23T05:42:15,966 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:15,966 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:15,967 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:15,968 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into 74b78433e0234547ba1ae7634f2139a8(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:15,968 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:15,968 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=13, startTime=1732340535934; duration=0sec 2024-11-23T05:42:15,968 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:15,968 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:15,968 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36795 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:15,968 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:15,968 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:15,969 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/0278dedfe40749fa9c5acbe99210dd27, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a23a0287a09e424ca2ae29a743b744d8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4b5786e738043bb817218de4a879f64] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=35.9 K 2024-11-23T05:42:15,969 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 0278dedfe40749fa9c5acbe99210dd27, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=154, earliestPutTs=1732340530471 2024-11-23T05:42:15,969 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a23a0287a09e424ca2ae29a743b744d8, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=169, earliestPutTs=1732340531643 2024-11-23T05:42:15,969 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d4b5786e738043bb817218de4a879f64, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732340533835 2024-11-23T05:42:15,976 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#391 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:15,977 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/dded319cbc1f474488a597058c335255 is 50, key is test_row_0/C:col10/1732340533835/Put/seqid=0 2024-11-23T05:42:15,983 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742289_1465 (size=12595) 2024-11-23T05:42:16,019 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:16,019 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:16,020 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:42:16,024 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/f47534ee2ff04597a76947fa3ed1f1f9 is 50, key is test_row_0/A:col10/1732340536018/Put/seqid=0 2024-11-23T05:42:16,027 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742290_1466 (size=14541) 2024-11-23T05:42:16,027 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/f47534ee2ff04597a76947fa3ed1f1f9 2024-11-23T05:42:16,033 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/40731f0c3cdc46cb89bcebdf3371c2ac is 50, key is test_row_0/B:col10/1732340536018/Put/seqid=0 2024-11-23T05:42:16,035 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742291_1467 (size=12151) 2024-11-23T05:42:16,039 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:16,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:16,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,039 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,039 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,040 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,055 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T05:42:16,091 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,091 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340596083, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,091 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340596084, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,093 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,093 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340596085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,093 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340596196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340596196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,203 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,203 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340596196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,203 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,204 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,387 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/dded319cbc1f474488a597058c335255 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/dded319cbc1f474488a597058c335255 2024-11-23T05:42:16,390 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into dded319cbc1f474488a597058c335255(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:16,390 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:16,390 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=13, startTime=1732340535934; duration=0sec 2024-11-23T05:42:16,390 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:16,390 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:16,410 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340596405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,411 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,411 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340596405, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,411 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,421 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,421 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 76 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340596416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,422 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,436 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/40731f0c3cdc46cb89bcebdf3371c2ac 2024-11-23T05:42:16,441 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/8790993497374499b2bd03c491473dc5 is 50, key is test_row_0/C:col10/1732340536018/Put/seqid=0 2024-11-23T05:42:16,444 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742292_1468 (size=12151) 2024-11-23T05:42:16,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:16,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:16,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,499 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T05:42:16,651 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:16,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:16,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,718 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340596716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,718 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,719 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340596716, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,719 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,726 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,727 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340596726, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,727 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,804 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,845 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=208 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/8790993497374499b2bd03c491473dc5 2024-11-23T05:42:16,848 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/f47534ee2ff04597a76947fa3ed1f1f9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/f47534ee2ff04597a76947fa3ed1f1f9 2024-11-23T05:42:16,850 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/f47534ee2ff04597a76947fa3ed1f1f9, entries=200, sequenceid=208, filesize=14.2 K 2024-11-23T05:42:16,851 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/40731f0c3cdc46cb89bcebdf3371c2ac as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/40731f0c3cdc46cb89bcebdf3371c2ac 2024-11-23T05:42:16,854 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/40731f0c3cdc46cb89bcebdf3371c2ac, entries=150, sequenceid=208, filesize=11.9 K 2024-11-23T05:42:16,855 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/8790993497374499b2bd03c491473dc5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/8790993497374499b2bd03c491473dc5 2024-11-23T05:42:16,858 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/8790993497374499b2bd03c491473dc5, entries=150, sequenceid=208, filesize=11.9 K 2024-11-23T05:42:16,859 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96da1e6d03a7b375833ea586fe9dc796 in 840ms, sequenceid=208, compaction requested=false 2024-11-23T05:42:16,859 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:16,897 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:16,897 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:16,897 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=147.60 KB heapSize=387.47 KB 2024-11-23T05:42:16,901 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/88e8ea6e52234da38010ccaa6ad767c2 is 50, key is test_row_0/A:col10/1732340536896/Put/seqid=0 2024-11-23T05:42:16,904 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742293_1469 (size=14541) 2024-11-23T05:42:16,924 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,924 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340596916, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,924 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,924 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:16,925 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340596919, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:16,925 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:16,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:16,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:16,956 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,956 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:16,956 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,031 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,031 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340597026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,031 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,043 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,043 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340597036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,043 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,108 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:17,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:17,108 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,108 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,109 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340597226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,233 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,233 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340597226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,233 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340597236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,242 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,242 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340597236, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,242 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,243 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,251 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340597246, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,251 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,261 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,261 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,261 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,262 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,305 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/88e8ea6e52234da38010ccaa6ad767c2 2024-11-23T05:42:17,310 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/5361dccdc8f444a9923b846bc6bde0b4 is 50, key is test_row_0/B:col10/1732340536896/Put/seqid=0 2024-11-23T05:42:17,313 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742294_1470 (size=12151) 2024-11-23T05:42:17,414 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,414 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,414 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,415 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340597556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,559 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:17,559 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340597556, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:17,559 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:17,567 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,567 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,568 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,576 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T05:42:17,713 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/5361dccdc8f444a9923b846bc6bde0b4 2024-11-23T05:42:17,718 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/4bac736adb524594934bace63b95edd6 is 50, key is test_row_0/C:col10/1732340536896/Put/seqid=0 2024-11-23T05:42:17,719 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:17,719 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:17,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,720 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,720 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,720 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,722 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742295_1471 (size=12151) 2024-11-23T05:42:17,872 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:17,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:17,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:17,872 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,872 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:17,873 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:18,025 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:18,025 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] handler.RSProcedureHandler(58): pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:18,025 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=130 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:18,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=130 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:18,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340598065, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,070 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,071 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340598066, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,071 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,123 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=49.20 KB at sequenceid=234 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/4bac736adb524594934bace63b95edd6 2024-11-23T05:42:18,126 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/88e8ea6e52234da38010ccaa6ad767c2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/88e8ea6e52234da38010ccaa6ad767c2 2024-11-23T05:42:18,128 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/88e8ea6e52234da38010ccaa6ad767c2, entries=200, sequenceid=234, filesize=14.2 K 2024-11-23T05:42:18,129 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/5361dccdc8f444a9923b846bc6bde0b4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5361dccdc8f444a9923b846bc6bde0b4 2024-11-23T05:42:18,131 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5361dccdc8f444a9923b846bc6bde0b4, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T05:42:18,132 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/4bac736adb524594934bace63b95edd6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/4bac736adb524594934bace63b95edd6 2024-11-23T05:42:18,134 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/4bac736adb524594934bace63b95edd6, entries=150, sequenceid=234, filesize=11.9 K 2024-11-23T05:42:18,135 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~147.60 KB/151140, heapSize ~387.42 KB/396720, currentSize=53.67 KB/54960 for 96da1e6d03a7b375833ea586fe9dc796 in 1238ms, sequenceid=234, compaction requested=true 2024-11-23T05:42:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:18,135 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:18,135 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:18,135 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:18,135 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 41677 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:18,135 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:18,136 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:18,136 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/74b78433e0234547ba1ae7634f2139a8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/f47534ee2ff04597a76947fa3ed1f1f9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/88e8ea6e52234da38010ccaa6ad767c2] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=40.7 K 2024-11-23T05:42:18,136 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 74b78433e0234547ba1ae7634f2139a8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732340533835 2024-11-23T05:42:18,136 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting f47534ee2ff04597a76947fa3ed1f1f9, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732340534706 2024-11-23T05:42:18,136 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:18,136 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:18,136 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 88e8ea6e52234da38010ccaa6ad767c2, keycount=200, bloomtype=ROW, size=14.2 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732340536082 2024-11-23T05:42:18,136 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:18,136 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6eb1a433e93446eab19ca15dfc1559f8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/40731f0c3cdc46cb89bcebdf3371c2ac, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5361dccdc8f444a9923b846bc6bde0b4] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=36.0 K 2024-11-23T05:42:18,137 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6eb1a433e93446eab19ca15dfc1559f8, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732340533835 2024-11-23T05:42:18,137 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 40731f0c3cdc46cb89bcebdf3371c2ac, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732340534706 2024-11-23T05:42:18,137 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5361dccdc8f444a9923b846bc6bde0b4, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732340536082 2024-11-23T05:42:18,144 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#398 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:18,145 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/49a448c9ca01406bbcff6fdcd2911c4b is 50, key is test_row_0/A:col10/1732340536896/Put/seqid=0 2024-11-23T05:42:18,146 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#399 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:18,146 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6bc53c5fe0bb4844805b148809357bba is 50, key is test_row_0/B:col10/1732340536896/Put/seqid=0 2024-11-23T05:42:18,151 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742296_1472 (size=12697) 2024-11-23T05:42:18,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742297_1473 (size=12697) 2024-11-23T05:42:18,177 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=130 2024-11-23T05:42:18,177 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:18,177 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:42:18,181 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/7c3bc185865240938786fa0a68dff44a is 50, key is test_row_0/A:col10/1732340536915/Put/seqid=0 2024-11-23T05:42:18,184 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742298_1474 (size=12151) 2024-11-23T05:42:18,185 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/7c3bc185865240938786fa0a68dff44a 2024-11-23T05:42:18,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/3ecfe51139e4469e8c915bd36d0a4a10 is 50, key is test_row_0/B:col10/1732340536915/Put/seqid=0 2024-11-23T05:42:18,195 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742299_1475 (size=12151) 2024-11-23T05:42:18,253 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:18,253 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:18,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340598302, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340598303, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,313 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,313 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340598305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,313 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340598416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340598416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,428 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340598416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,428 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,555 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6bc53c5fe0bb4844805b148809357bba as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6bc53c5fe0bb4844805b148809357bba 2024-11-23T05:42:18,555 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/49a448c9ca01406bbcff6fdcd2911c4b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/49a448c9ca01406bbcff6fdcd2911c4b 2024-11-23T05:42:18,558 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into 49a448c9ca01406bbcff6fdcd2911c4b(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:18,558 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into 6bc53c5fe0bb4844805b148809357bba(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:18,558 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:18,558 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:18,558 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=13, startTime=1732340538135; duration=0sec 2024-11-23T05:42:18,558 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=13, startTime=1732340538135; duration=0sec 2024-11-23T05:42:18,558 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:18,558 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:18,558 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:18,558 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:18,558 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:18,559 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36897 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:18,559 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:18,559 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:18,559 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/dded319cbc1f474488a597058c335255, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/8790993497374499b2bd03c491473dc5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/4bac736adb524594934bace63b95edd6] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=36.0 K 2024-11-23T05:42:18,559 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting dded319cbc1f474488a597058c335255, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=194, earliestPutTs=1732340533835 2024-11-23T05:42:18,560 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 8790993497374499b2bd03c491473dc5, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=208, earliestPutTs=1732340534706 2024-11-23T05:42:18,560 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 4bac736adb524594934bace63b95edd6, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732340536082 2024-11-23T05:42:18,566 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#402 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:18,566 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e39894d757ef48179b86bc56f3cb617f is 50, key is test_row_0/C:col10/1732340536896/Put/seqid=0 2024-11-23T05:42:18,571 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742300_1476 (size=12697) 2024-11-23T05:42:18,596 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/3ecfe51139e4469e8c915bd36d0a4a10 2024-11-23T05:42:18,603 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/2ffc7397712c4b9cabdf70eb0207a048 is 50, key is test_row_0/C:col10/1732340536915/Put/seqid=0 2024-11-23T05:42:18,612 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742301_1477 (size=12151) 2024-11-23T05:42:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340598636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340598636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,642 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,642 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,642 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340598636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,643 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340598946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 88 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340598946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,952 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,952 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340598946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:18,952 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:18,953 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:18,974 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/e39894d757ef48179b86bc56f3cb617f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e39894d757ef48179b86bc56f3cb617f 2024-11-23T05:42:18,979 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into e39894d757ef48179b86bc56f3cb617f(size=12.4 K), total size for store is 12.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:18,979 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:18,979 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=13, startTime=1732340538135; duration=0sec 2024-11-23T05:42:18,979 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:18,979 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:19,013 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=245 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/2ffc7397712c4b9cabdf70eb0207a048 2024-11-23T05:42:19,017 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/7c3bc185865240938786fa0a68dff44a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7c3bc185865240938786fa0a68dff44a 2024-11-23T05:42:19,020 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7c3bc185865240938786fa0a68dff44a, entries=150, sequenceid=245, filesize=11.9 K 2024-11-23T05:42:19,021 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/3ecfe51139e4469e8c915bd36d0a4a10 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3ecfe51139e4469e8c915bd36d0a4a10 2024-11-23T05:42:19,024 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3ecfe51139e4469e8c915bd36d0a4a10, entries=150, sequenceid=245, filesize=11.9 K 2024-11-23T05:42:19,024 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/2ffc7397712c4b9cabdf70eb0207a048 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/2ffc7397712c4b9cabdf70eb0207a048 2024-11-23T05:42:19,027 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/2ffc7397712c4b9cabdf70eb0207a048, entries=150, sequenceid=245, filesize=11.9 K 2024-11-23T05:42:19,028 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for 96da1e6d03a7b375833ea586fe9dc796 in 851ms, sequenceid=245, compaction requested=false 2024-11-23T05:42:19,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:19,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:19,028 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=130}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=130 2024-11-23T05:42:19,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=130 2024-11-23T05:42:19,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=130, resume processing ppid=129 2024-11-23T05:42:19,030 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=130, ppid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.6000 sec 2024-11-23T05:42:19,032 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=129, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=129, table=TestAcidGuarantees in 3.6040 sec 2024-11-23T05:42:19,080 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:19,080 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:19,080 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:42:19,085 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/5be0ddd76d314483b387a3c9753dbeed is 50, key is test_row_0/A:col10/1732340538297/Put/seqid=0 2024-11-23T05:42:19,088 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742302_1478 (size=14741) 2024-11-23T05:42:19,110 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,110 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340599101, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,110 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,118 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340599109, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,119 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,223 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,223 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340599216, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,223 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,232 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,232 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340599226, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,232 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,427 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,427 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340599426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,427 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,442 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,442 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340599436, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,442 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340599455, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,462 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 89 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340599456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,462 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,471 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,471 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 99 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340599466, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,471 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,488 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/5be0ddd76d314483b387a3c9753dbeed 2024-11-23T05:42:19,495 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/962790488dd840e3a735b1389589dd07 is 50, key is test_row_0/B:col10/1732340538297/Put/seqid=0 2024-11-23T05:42:19,499 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742303_1479 (size=12301) 2024-11-23T05:42:19,501 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/962790488dd840e3a735b1389589dd07 2024-11-23T05:42:19,507 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/cc6b4a3c840640a1972cf7eec4bce8e9 is 50, key is test_row_0/C:col10/1732340538297/Put/seqid=0 2024-11-23T05:42:19,520 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742304_1480 (size=12301) 2024-11-23T05:42:19,586 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=129 2024-11-23T05:42:19,586 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:19,587 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:19,587 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees 2024-11-23T05:42:19,588 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:19,588 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T05:42:19,588 INFO [PEWorker-5 {}] procedure.FlushTableProcedure(91): pid=131, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:19,589 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=132, ppid=131, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:19,696 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T05:42:19,740 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T05:42:19,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:19,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:19,740 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:19,740 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:19,741 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:19,741 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:19,741 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340599736, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,742 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,750 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:19,750 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340599745, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:19,751 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:19,893 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T05:42:19,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:19,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:19,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:19,893 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] handler.RSProcedureHandler(58): pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:19,893 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=132 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:19,894 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=132 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:19,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T05:42:19,921 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=274 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/cc6b4a3c840640a1972cf7eec4bce8e9 2024-11-23T05:42:19,924 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/5be0ddd76d314483b387a3c9753dbeed as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5be0ddd76d314483b387a3c9753dbeed 2024-11-23T05:42:19,927 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5be0ddd76d314483b387a3c9753dbeed, entries=200, sequenceid=274, filesize=14.4 K 2024-11-23T05:42:19,927 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/962790488dd840e3a735b1389589dd07 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/962790488dd840e3a735b1389589dd07 2024-11-23T05:42:19,930 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/962790488dd840e3a735b1389589dd07, entries=150, sequenceid=274, filesize=12.0 K 2024-11-23T05:42:19,931 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/cc6b4a3c840640a1972cf7eec4bce8e9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cc6b4a3c840640a1972cf7eec4bce8e9 2024-11-23T05:42:19,934 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cc6b4a3c840640a1972cf7eec4bce8e9, entries=150, sequenceid=274, filesize=12.0 K 2024-11-23T05:42:19,935 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=46.96 KB/48090 for 96da1e6d03a7b375833ea586fe9dc796 in 854ms, sequenceid=274, compaction requested=true 2024-11-23T05:42:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:19,935 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:19,935 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:19,935 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:19,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39589 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:19,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:19,936 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:19,936 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/49a448c9ca01406bbcff6fdcd2911c4b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7c3bc185865240938786fa0a68dff44a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5be0ddd76d314483b387a3c9753dbeed] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=38.7 K 2024-11-23T05:42:19,936 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:19,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 49a448c9ca01406bbcff6fdcd2911c4b, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732340536082 2024-11-23T05:42:19,936 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:19,936 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:19,936 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6bc53c5fe0bb4844805b148809357bba, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3ecfe51139e4469e8c915bd36d0a4a10, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/962790488dd840e3a735b1389589dd07] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=36.3 K 2024-11-23T05:42:19,936 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7c3bc185865240938786fa0a68dff44a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732340536914 2024-11-23T05:42:19,937 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bc53c5fe0bb4844805b148809357bba, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732340536082 2024-11-23T05:42:19,937 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5be0ddd76d314483b387a3c9753dbeed, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732340538297 2024-11-23T05:42:19,937 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3ecfe51139e4469e8c915bd36d0a4a10, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732340536914 2024-11-23T05:42:19,937 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 962790488dd840e3a735b1389589dd07, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732340538297 2024-11-23T05:42:19,945 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#407 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:19,945 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/ce90503f0a9e4a35bc24bc837f35273c is 50, key is test_row_0/A:col10/1732340538297/Put/seqid=0 2024-11-23T05:42:19,948 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#408 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:19,948 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/c67ce7959b2a44d0921b9c485e9c0cb4 is 50, key is test_row_0/B:col10/1732340538297/Put/seqid=0 2024-11-23T05:42:19,951 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742305_1481 (size=12949) 2024-11-23T05:42:19,963 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742306_1482 (size=12949) 2024-11-23T05:42:20,045 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=132 2024-11-23T05:42:20,045 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:20,045 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=46.96 KB heapSize=123.80 KB 2024-11-23T05:42:20,049 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/5c4ba999087d4229a6c56820e6ea90bc is 50, key is test_row_0/A:col10/1732340539107/Put/seqid=0 2024-11-23T05:42:20,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742307_1483 (size=12301) 2024-11-23T05:42:20,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T05:42:20,251 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:20,251 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:20,339 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340600330, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,339 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,340 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 95 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340600339, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,347 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,355 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/ce90503f0a9e4a35bc24bc837f35273c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ce90503f0a9e4a35bc24bc837f35273c 2024-11-23T05:42:20,357 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into ce90503f0a9e4a35bc24bc837f35273c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:20,358 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:20,358 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=13, startTime=1732340539935; duration=0sec 2024-11-23T05:42:20,358 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:20,358 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:20,358 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:20,358 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37149 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:20,358 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:20,358 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:20,358 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e39894d757ef48179b86bc56f3cb617f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/2ffc7397712c4b9cabdf70eb0207a048, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cc6b4a3c840640a1972cf7eec4bce8e9] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=36.3 K 2024-11-23T05:42:20,359 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting e39894d757ef48179b86bc56f3cb617f, keycount=150, bloomtype=ROW, size=12.4 K, encoding=NONE, compression=NONE, seqNum=234, earliestPutTs=1732340536082 2024-11-23T05:42:20,359 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2ffc7397712c4b9cabdf70eb0207a048, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=245, earliestPutTs=1732340536914 2024-11-23T05:42:20,359 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting cc6b4a3c840640a1972cf7eec4bce8e9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732340538297 2024-11-23T05:42:20,364 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#410 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:20,365 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/923b4055a4f244ee8b536ede5faa074c is 50, key is test_row_0/C:col10/1732340538297/Put/seqid=0 2024-11-23T05:42:20,367 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/c67ce7959b2a44d0921b9c485e9c0cb4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/c67ce7959b2a44d0921b9c485e9c0cb4 2024-11-23T05:42:20,367 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742308_1484 (size=12949) 2024-11-23T05:42:20,370 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into c67ce7959b2a44d0921b9c485e9c0cb4(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:20,370 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:20,370 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=13, startTime=1732340539935; duration=0sec 2024-11-23T05:42:20,370 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:20,370 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:20,451 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,451 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340600446, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,451 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,452 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/5c4ba999087d4229a6c56820e6ea90bc 2024-11-23T05:42:20,459 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/5d986d622a4f47079541bc68a77b47be is 50, key is test_row_0/B:col10/1732340539107/Put/seqid=0 2024-11-23T05:42:20,461 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,462 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 96 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340600456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,462 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,463 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742309_1485 (size=12301) 2024-11-23T05:42:20,481 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,481 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 92 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340600476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 90 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340600476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,482 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,482 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340600476, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,482 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2180 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:20,482 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2179 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,482 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,483 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2177 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,483 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,662 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340600656, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,663 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,671 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,671 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 97 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340600666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,671 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,726 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T05:42:20,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/923b4055a4f244ee8b536ede5faa074c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/923b4055a4f244ee8b536ede5faa074c 2024-11-23T05:42:20,775 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into 923b4055a4f244ee8b536ede5faa074c(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:20,775 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:20,775 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=13, startTime=1732340539935; duration=0sec 2024-11-23T05:42:20,775 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:20,775 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:20,864 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/5d986d622a4f47079541bc68a77b47be 2024-11-23T05:42:20,869 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/907db295ca5749aa965792fdab0ee094 is 50, key is test_row_0/C:col10/1732340539107/Put/seqid=0 2024-11-23T05:42:20,876 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742310_1486 (size=12301) 2024-11-23T05:42:20,967 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 94 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340600965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,968 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:20,981 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:20,981 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 98 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340600976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:20,981 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:21,276 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=15.65 KB at sequenceid=284 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/907db295ca5749aa965792fdab0ee094 2024-11-23T05:42:21,321 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/5c4ba999087d4229a6c56820e6ea90bc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5c4ba999087d4229a6c56820e6ea90bc 2024-11-23T05:42:21,325 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5c4ba999087d4229a6c56820e6ea90bc, entries=150, sequenceid=284, filesize=12.0 K 2024-11-23T05:42:21,327 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/5d986d622a4f47079541bc68a77b47be as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5d986d622a4f47079541bc68a77b47be 2024-11-23T05:42:21,331 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5d986d622a4f47079541bc68a77b47be, entries=150, sequenceid=284, filesize=12.0 K 2024-11-23T05:42:21,331 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/907db295ca5749aa965792fdab0ee094 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/907db295ca5749aa965792fdab0ee094 2024-11-23T05:42:21,337 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/907db295ca5749aa965792fdab0ee094, entries=150, sequenceid=284, filesize=12.0 K 2024-11-23T05:42:21,338 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(3140): Finished flush of dataSize ~46.96 KB/48090, heapSize ~123.75 KB/126720, currentSize=154.31 KB/158010 for 96da1e6d03a7b375833ea586fe9dc796 in 1293ms, sequenceid=284, compaction requested=false 2024-11-23T05:42:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:21,339 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=132}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=132 2024-11-23T05:42:21,339 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=132 2024-11-23T05:42:21,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=132, resume processing ppid=131 2024-11-23T05:42:21,342 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=132, ppid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7510 sec 2024-11-23T05:42:21,346 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=131, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=131, table=TestAcidGuarantees in 1.7580 sec 2024-11-23T05:42:21,480 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:21,480 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:21,481 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=161.02 KB heapSize=422.63 KB 2024-11-23T05:42:21,485 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/65e26b0a28cb4c59adccb7f8918abb40 is 50, key is test_row_0/A:col10/1732340540325/Put/seqid=0 2024-11-23T05:42:21,488 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742311_1487 (size=14741) 2024-11-23T05:42:21,510 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:21,510 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340601503, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,510 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:21,513 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:21,513 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 100 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340601507, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,513 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:21,627 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340601616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:21,627 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,628 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340601616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,628 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,629 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:21,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=131 2024-11-23T05:42:21,737 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:21,739 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:21,740 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees 2024-11-23T05:42:21,742 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T05:42:21,743 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:21,744 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=133, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:21,744 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=134, ppid=133, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:21,842 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340601836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:21,843 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:21,843 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 102 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340601836, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:21,843 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:21,845 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T05:42:21,889 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/65e26b0a28cb4c59adccb7f8918abb40 2024-11-23T05:42:21,894 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/a712c94c09784945842da96358501e6f is 50, key is test_row_0/B:col10/1732340540325/Put/seqid=0 2024-11-23T05:42:21,895 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T05:42:21,895 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:21,896 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:21,896 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:21,896 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:21,898 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742312_1488 (size=12301) 2024-11-23T05:42:22,048 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T05:42:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,048 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,048 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,056 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T05:42:22,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340602156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:22,161 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,161 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 103 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340602156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,161 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,162 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:22,200 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T05:42:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,201 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,201 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,201 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,299 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/a712c94c09784945842da96358501e6f 2024-11-23T05:42:22,309 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/287751cb38104349aa289721dce447d9 is 50, key is test_row_0/C:col10/1732340540325/Put/seqid=0 2024-11-23T05:42:22,314 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742313_1489 (size=12301) 2024-11-23T05:42:22,347 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T05:42:22,353 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T05:42:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,353 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,353 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,366 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T05:42:22,501 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 93 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47688 deadline: 1732340602496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:22,502 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:22,502 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 91 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47686 deadline: 1732340602496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:22,502 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 101 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47754 deadline: 1732340602496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:22,502 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4200 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:22,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,502 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:22,502 WARN [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4199 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:22,503 WARN [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in 96da1e6d03a7b375833ea586fe9dc796 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4197 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,503 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:22,505 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T05:42:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,505 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,505 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,658 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T05:42:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,658 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] handler.RSProcedureHandler(58): pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,658 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=134 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,659 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=134 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:22,673 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340602666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,674 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:22,674 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 104 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340602666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:22,674 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:22,714 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=53.67 KB at sequenceid=314 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/287751cb38104349aa289721dce447d9 2024-11-23T05:42:22,718 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/65e26b0a28cb4c59adccb7f8918abb40 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/65e26b0a28cb4c59adccb7f8918abb40 2024-11-23T05:42:22,720 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/65e26b0a28cb4c59adccb7f8918abb40, entries=200, sequenceid=314, filesize=14.4 K 2024-11-23T05:42:22,721 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/a712c94c09784945842da96358501e6f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/a712c94c09784945842da96358501e6f 2024-11-23T05:42:22,724 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/a712c94c09784945842da96358501e6f, entries=150, sequenceid=314, filesize=12.0 K 2024-11-23T05:42:22,724 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/287751cb38104349aa289721dce447d9 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/287751cb38104349aa289721dce447d9 2024-11-23T05:42:22,727 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/287751cb38104349aa289721dce447d9, entries=150, sequenceid=314, filesize=12.0 K 2024-11-23T05:42:22,728 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~161.02 KB/164880, heapSize ~422.58 KB/432720, currentSize=40.25 KB/41220 for 96da1e6d03a7b375833ea586fe9dc796 in 1247ms, sequenceid=314, compaction requested=true 2024-11-23T05:42:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:22,728 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:22,728 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:22,728 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:22,729 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 39991 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:22,729 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:22,729 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,730 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ce90503f0a9e4a35bc24bc837f35273c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5c4ba999087d4229a6c56820e6ea90bc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/65e26b0a28cb4c59adccb7f8918abb40] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=39.1 K 2024-11-23T05:42:22,730 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:22,730 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:22,730 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,730 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/c67ce7959b2a44d0921b9c485e9c0cb4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5d986d622a4f47079541bc68a77b47be, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/a712c94c09784945842da96358501e6f] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=36.7 K 2024-11-23T05:42:22,730 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ce90503f0a9e4a35bc24bc837f35273c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732340538297 2024-11-23T05:42:22,730 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting c67ce7959b2a44d0921b9c485e9c0cb4, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732340538297 2024-11-23T05:42:22,730 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 5c4ba999087d4229a6c56820e6ea90bc, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732340539091 2024-11-23T05:42:22,730 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d986d622a4f47079541bc68a77b47be, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732340539091 2024-11-23T05:42:22,731 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting a712c94c09784945842da96358501e6f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732340540325 2024-11-23T05:42:22,731 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 65e26b0a28cb4c59adccb7f8918abb40, keycount=200, bloomtype=ROW, size=14.4 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732340540325 2024-11-23T05:42:22,737 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#416 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:22,737 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/91a3da931e044a1eb5c0685e9905bf49 is 50, key is test_row_0/B:col10/1732340540325/Put/seqid=0 2024-11-23T05:42:22,737 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#417 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:22,737 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/d0dc1fd00f4444d2849c879cadfd9245 is 50, key is test_row_0/A:col10/1732340540325/Put/seqid=0 2024-11-23T05:42:22,740 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742314_1490 (size=13051) 2024-11-23T05:42:22,741 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742315_1491 (size=13051) 2024-11-23T05:42:22,744 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/d0dc1fd00f4444d2849c879cadfd9245 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/d0dc1fd00f4444d2849c879cadfd9245 2024-11-23T05:42:22,748 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into d0dc1fd00f4444d2849c879cadfd9245(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:22,748 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:22,748 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=13, startTime=1732340542728; duration=0sec 2024-11-23T05:42:22,748 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:22,748 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:22,748 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:22,748 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 37551 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:22,748 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:22,749 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,749 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/923b4055a4f244ee8b536ede5faa074c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/907db295ca5749aa965792fdab0ee094, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/287751cb38104349aa289721dce447d9] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=36.7 K 2024-11-23T05:42:22,749 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 923b4055a4f244ee8b536ede5faa074c, keycount=150, bloomtype=ROW, size=12.6 K, encoding=NONE, compression=NONE, seqNum=274, earliestPutTs=1732340538297 2024-11-23T05:42:22,749 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 907db295ca5749aa965792fdab0ee094, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=284, earliestPutTs=1732340539091 2024-11-23T05:42:22,749 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 287751cb38104349aa289721dce447d9, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732340540325 2024-11-23T05:42:22,755 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#418 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:22,755 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/22a2c25efdb0496d9ad4d8afe183b7c5 is 50, key is test_row_0/C:col10/1732340540325/Put/seqid=0 2024-11-23T05:42:22,759 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742316_1492 (size=13051) 2024-11-23T05:42:22,810 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=134 2024-11-23T05:42:22,811 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:22,811 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=40.25 KB heapSize=106.22 KB 2024-11-23T05:42:22,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/0c3fa94486a24f88bb5a7bf99456b9d4 is 50, key is test_row_0/A:col10/1732340541492/Put/seqid=0 2024-11-23T05:42:22,817 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742317_1493 (size=9857) 2024-11-23T05:42:22,876 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T05:42:23,144 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/91a3da931e044a1eb5c0685e9905bf49 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/91a3da931e044a1eb5c0685e9905bf49 2024-11-23T05:42:23,148 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into 91a3da931e044a1eb5c0685e9905bf49(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:23,148 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:23,148 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=13, startTime=1732340542728; duration=0sec 2024-11-23T05:42:23,148 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:23,148 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:23,173 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/22a2c25efdb0496d9ad4d8afe183b7c5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/22a2c25efdb0496d9ad4d8afe183b7c5 2024-11-23T05:42:23,177 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into 22a2c25efdb0496d9ad4d8afe183b7c5(size=12.7 K), total size for store is 12.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:23,177 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:23,177 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=13, startTime=1732340542728; duration=0sec 2024-11-23T05:42:23,177 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:23,177 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:23,218 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/0c3fa94486a24f88bb5a7bf99456b9d4 2024-11-23T05:42:23,223 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/2cae2ca63bcb4cfba6480d3bff68b4bb is 50, key is test_row_0/B:col10/1732340541492/Put/seqid=0 2024-11-23T05:42:23,227 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742318_1494 (size=9857) 2024-11-23T05:42:23,628 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/2cae2ca63bcb4cfba6480d3bff68b4bb 2024-11-23T05:42:23,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/3608618aba03484e82aa6963da8c243a is 50, key is test_row_0/C:col10/1732340541492/Put/seqid=0 2024-11-23T05:42:23,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742319_1495 (size=9857) 2024-11-23T05:42:23,636 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=13.42 KB at sequenceid=324 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/3608618aba03484e82aa6963da8c243a 2024-11-23T05:42:23,639 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/0c3fa94486a24f88bb5a7bf99456b9d4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/0c3fa94486a24f88bb5a7bf99456b9d4 2024-11-23T05:42:23,642 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/0c3fa94486a24f88bb5a7bf99456b9d4, entries=100, sequenceid=324, filesize=9.6 K 2024-11-23T05:42:23,642 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/2cae2ca63bcb4cfba6480d3bff68b4bb as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/2cae2ca63bcb4cfba6480d3bff68b4bb 2024-11-23T05:42:23,645 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/2cae2ca63bcb4cfba6480d3bff68b4bb, entries=100, sequenceid=324, filesize=9.6 K 2024-11-23T05:42:23,646 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/3608618aba03484e82aa6963da8c243a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/3608618aba03484e82aa6963da8c243a 2024-11-23T05:42:23,648 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/3608618aba03484e82aa6963da8c243a, entries=100, sequenceid=324, filesize=9.6 K 2024-11-23T05:42:23,649 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(3140): Finished flush of dataSize ~40.25 KB/41220, heapSize ~106.17 KB/108720, currentSize=0 B/0 for 96da1e6d03a7b375833ea586fe9dc796 in 838ms, sequenceid=324, compaction requested=false 2024-11-23T05:42:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:23,649 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=134}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=134 2024-11-23T05:42:23,650 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=134 2024-11-23T05:42:23,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=134, resume processing ppid=133 2024-11-23T05:42:23,652 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=134, ppid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.9060 sec 2024-11-23T05:42:23,654 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=133, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=133, table=TestAcidGuarantees in 1.9130 sec 2024-11-23T05:42:23,698 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:23,698 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:23,699 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:42:23,703 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/723fe65c68914ebc97ed74cdd131b1b7 is 50, key is test_row_1/A:col10/1732340543698/Put/seqid=0 2024-11-23T05:42:23,706 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742320_1496 (size=12297) 2024-11-23T05:42:23,761 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:23,761 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340603756, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,762 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:23,764 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:23,764 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340603757, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,765 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:23,872 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:23,872 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340603866, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,872 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:23,879 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:23,880 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340603875, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:23,880 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:23,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=133 2024-11-23T05:42:23,886 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:23,886 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:23,887 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees 2024-11-23T05:42:23,888 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T05:42:23,888 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:23,888 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=135, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:23,889 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=136, ppid=135, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:23,996 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T05:42:24,040 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T05:42:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:24,040 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,040 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,041 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,041 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,081 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:24,082 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340604076, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,082 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:24,089 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:24,089 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340604085, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,090 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:24,106 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/723fe65c68914ebc97ed74cdd131b1b7 2024-11-23T05:42:24,111 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6af7973235a5468b9ac85c4f51c738c5 is 50, key is test_row_1/B:col10/1732340543698/Put/seqid=0 2024-11-23T05:42:24,114 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742321_1497 (size=9857) 2024-11-23T05:42:24,193 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T05:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,193 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,193 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,194 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,205 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T05:42:24,345 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T05:42:24,345 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:24,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,346 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,346 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,346 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,390 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:24,391 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340604386, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,391 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:24,398 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:24,399 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340604396, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,399 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:24,498 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T05:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,498 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,498 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,514 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6af7973235a5468b9ac85c4f51c738c5 2024-11-23T05:42:24,516 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T05:42:24,520 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/7730aac076c24e23a02f7f137ed30945 is 50, key is test_row_1/C:col10/1732340543698/Put/seqid=0 2024-11-23T05:42:24,523 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742322_1498 (size=9857) 2024-11-23T05:42:24,650 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T05:42:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:24,650 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,651 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,651 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,651 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,803 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T05:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,803 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] handler.RSProcedureHandler(58): pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,803 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=136 java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,803 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=136 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:24,816 INFO [Thread-1929 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1929. 2024-11-23T05:42:24,816 DEBUG [Thread-1929 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:24,816 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,816 DEBUG [Thread-1929 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,816 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:24,818 INFO [Thread-1931 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1931. 2024-11-23T05:42:24,818 DEBUG [Thread-1931 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:24,818 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,818 DEBUG [Thread-1931 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,818 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:24,819 INFO [Thread-1933 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1933. 2024-11-23T05:42:24,819 DEBUG [Thread-1933 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:24,819 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,819 DEBUG [Thread-1933 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,819 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:24,823 INFO [Thread-1927 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1927. 2024-11-23T05:42:24,823 DEBUG [Thread-1927 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:24,823 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,823 DEBUG [Thread-1927 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,823 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:24,826 INFO [Thread-1925 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1925. 2024-11-23T05:42:24,826 DEBUG [Thread-1925 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicScanReader.workDone(AcidGuaranteesTestTool.java:302) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:24,826 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,826 DEBUG [Thread-1925 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:24,826 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:24,907 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47746 deadline: 1732340604907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:24,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 125 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:47724 deadline: 1732340604907, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 , the old value is region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:24,908 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., hostname=9abfc7c2569e,45719,1732340394120, seqNum=2 because the exception is null or not the one we care about 2024-11-23T05:42:24,924 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=338 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/7730aac076c24e23a02f7f137ed30945 2024-11-23T05:42:24,932 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/723fe65c68914ebc97ed74cdd131b1b7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/723fe65c68914ebc97ed74cdd131b1b7 2024-11-23T05:42:24,938 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/723fe65c68914ebc97ed74cdd131b1b7, entries=150, sequenceid=338, filesize=12.0 K 2024-11-23T05:42:24,939 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/6af7973235a5468b9ac85c4f51c738c5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6af7973235a5468b9ac85c4f51c738c5 2024-11-23T05:42:24,943 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6af7973235a5468b9ac85c4f51c738c5, entries=100, sequenceid=338, filesize=9.6 K 2024-11-23T05:42:24,944 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/7730aac076c24e23a02f7f137ed30945 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/7730aac076c24e23a02f7f137ed30945 2024-11-23T05:42:24,948 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/7730aac076c24e23a02f7f137ed30945, entries=100, sequenceid=338, filesize=9.6 K 2024-11-23T05:42:24,949 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for 96da1e6d03a7b375833ea586fe9dc796 in 1250ms, sequenceid=338, compaction requested=true 2024-11-23T05:42:24,949 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:24,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:24,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:24,949 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:24,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:24,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:24,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store 96da1e6d03a7b375833ea586fe9dc796:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:24,949 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:24,949 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:24,950 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 35205 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:24,950 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:24,950 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/B is initiating minor compaction (all files) 2024-11-23T05:42:24,950 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/A is initiating minor compaction (all files) 2024-11-23T05:42:24,950 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/B in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,950 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/A in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,950 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/d0dc1fd00f4444d2849c879cadfd9245, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/0c3fa94486a24f88bb5a7bf99456b9d4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/723fe65c68914ebc97ed74cdd131b1b7] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=34.4 K 2024-11-23T05:42:24,950 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/91a3da931e044a1eb5c0685e9905bf49, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/2cae2ca63bcb4cfba6480d3bff68b4bb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6af7973235a5468b9ac85c4f51c738c5] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=32.0 K 2024-11-23T05:42:24,951 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 91a3da931e044a1eb5c0685e9905bf49, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732340540325 2024-11-23T05:42:24,951 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d0dc1fd00f4444d2849c879cadfd9245, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732340540325 2024-11-23T05:42:24,951 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0c3fa94486a24f88bb5a7bf99456b9d4, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732340541492 2024-11-23T05:42:24,951 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 2cae2ca63bcb4cfba6480d3bff68b4bb, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732340541492 2024-11-23T05:42:24,952 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 723fe65c68914ebc97ed74cdd131b1b7, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732340543693 2024-11-23T05:42:24,952 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6af7973235a5468b9ac85c4f51c738c5, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732340543698 2024-11-23T05:42:24,955 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=136 2024-11-23T05:42:24,955 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:24,956 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:42:24,959 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#B#compaction#425 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:24,960 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/571aaab513594547b8f8674512d9a5f4 is 50, key is test_row_0/B:col10/1732340541492/Put/seqid=0 2024-11-23T05:42:24,960 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#A#compaction#426 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:24,960 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/6ee7b5fccab6415b9c998a156dbc30d8 is 50, key is test_row_0/A:col10/1732340541492/Put/seqid=0 2024-11-23T05:42:24,960 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/98c73615edee4b798f7e9157490bafe6 is 50, key is test_row_0/A:col10/1732340543756/Put/seqid=0 2024-11-23T05:42:24,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742323_1499 (size=13153) 2024-11-23T05:42:24,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742324_1500 (size=12301) 2024-11-23T05:42:24,969 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742325_1501 (size=13153) 2024-11-23T05:42:25,026 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T05:42:25,366 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/98c73615edee4b798f7e9157490bafe6 2024-11-23T05:42:25,375 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/571aaab513594547b8f8674512d9a5f4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/571aaab513594547b8f8674512d9a5f4 2024-11-23T05:42:25,377 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/6ee7b5fccab6415b9c998a156dbc30d8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/6ee7b5fccab6415b9c998a156dbc30d8 2024-11-23T05:42:25,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/57f8575ed88f490ab6ec53472508ae67 is 50, key is test_row_0/B:col10/1732340543756/Put/seqid=0 2024-11-23T05:42:25,382 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/A of 96da1e6d03a7b375833ea586fe9dc796 into 6ee7b5fccab6415b9c998a156dbc30d8(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:25,382 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/B of 96da1e6d03a7b375833ea586fe9dc796 into 571aaab513594547b8f8674512d9a5f4(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:25,382 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:25,382 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:25,382 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/A, priority=13, startTime=1732340544949; duration=0sec 2024-11-23T05:42:25,382 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/B, priority=13, startTime=1732340544949; duration=0sec 2024-11-23T05:42:25,382 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:25,382 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:25,382 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:A 2024-11-23T05:42:25,382 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:B 2024-11-23T05:42:25,382 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:25,383 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742326_1502 (size=12301) 2024-11-23T05:42:25,383 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 32765 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:25,383 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): 96da1e6d03a7b375833ea586fe9dc796/C is initiating minor compaction (all files) 2024-11-23T05:42:25,383 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of 96da1e6d03a7b375833ea586fe9dc796/C in TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:25,383 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/22a2c25efdb0496d9ad4d8afe183b7c5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/3608618aba03484e82aa6963da8c243a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/7730aac076c24e23a02f7f137ed30945] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp, totalSize=32.0 K 2024-11-23T05:42:25,383 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 22a2c25efdb0496d9ad4d8afe183b7c5, keycount=150, bloomtype=ROW, size=12.7 K, encoding=NONE, compression=NONE, seqNum=314, earliestPutTs=1732340540325 2024-11-23T05:42:25,384 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3608618aba03484e82aa6963da8c243a, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=324, earliestPutTs=1732340541492 2024-11-23T05:42:25,384 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 7730aac076c24e23a02f7f137ed30945, keycount=100, bloomtype=ROW, size=9.6 K, encoding=NONE, compression=NONE, seqNum=338, earliestPutTs=1732340543698 2024-11-23T05:42:25,388 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): 96da1e6d03a7b375833ea586fe9dc796#C#compaction#429 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:25,388 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/115666f2df094041aa1be25619cb9782 is 50, key is test_row_0/C:col10/1732340541492/Put/seqid=0 2024-11-23T05:42:25,391 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742327_1503 (size=13153) 2024-11-23T05:42:25,784 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/57f8575ed88f490ab6ec53472508ae67 2024-11-23T05:42:25,791 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/c7783b31bd1d4c398f337ac351b934ae is 50, key is test_row_0/C:col10/1732340543756/Put/seqid=0 2024-11-23T05:42:25,793 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742328_1504 (size=12301) 2024-11-23T05:42:25,795 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/115666f2df094041aa1be25619cb9782 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/115666f2df094041aa1be25619cb9782 2024-11-23T05:42:25,798 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in 96da1e6d03a7b375833ea586fe9dc796/C of 96da1e6d03a7b375833ea586fe9dc796 into 115666f2df094041aa1be25619cb9782(size=12.8 K), total size for store is 12.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:25,799 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:25,799 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., storeName=96da1e6d03a7b375833ea586fe9dc796/C, priority=13, startTime=1732340544949; duration=0sec 2024-11-23T05:42:25,799 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:25,799 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: 96da1e6d03a7b375833ea586fe9dc796:C 2024-11-23T05:42:25,917 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:25,917 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. as already flushing 2024-11-23T05:42:25,917 INFO [Thread-1920 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1920. 2024-11-23T05:42:25,917 DEBUG [Thread-1920 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:25,917 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:25,917 DEBUG [Thread-1920 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:25,917 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:25,927 INFO [Thread-1918 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1918. 2024-11-23T05:42:25,927 DEBUG [Thread-1918 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:25,927 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:25,927 DEBUG [Thread-1918 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:25,927 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:26,037 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T05:42:26,194 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=362 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/c7783b31bd1d4c398f337ac351b934ae 2024-11-23T05:42:26,200 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/98c73615edee4b798f7e9157490bafe6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/98c73615edee4b798f7e9157490bafe6 2024-11-23T05:42:26,206 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/98c73615edee4b798f7e9157490bafe6, entries=150, sequenceid=362, filesize=12.0 K 2024-11-23T05:42:26,207 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/57f8575ed88f490ab6ec53472508ae67 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/57f8575ed88f490ab6ec53472508ae67 2024-11-23T05:42:26,212 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/57f8575ed88f490ab6ec53472508ae67, entries=150, sequenceid=362, filesize=12.0 K 2024-11-23T05:42:26,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/c7783b31bd1d4c398f337ac351b934ae as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/c7783b31bd1d4c398f337ac351b934ae 2024-11-23T05:42:26,216 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/c7783b31bd1d4c398f337ac351b934ae, entries=150, sequenceid=362, filesize=12.0 K 2024-11-23T05:42:26,217 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=13.42 KB/13740 for 96da1e6d03a7b375833ea586fe9dc796 in 1262ms, sequenceid=362, compaction requested=false 2024-11-23T05:42:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.HRegion(2603): Flush status journal for 96da1e6d03a7b375833ea586fe9dc796: 2024-11-23T05:42:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:26,217 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=136}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=136 2024-11-23T05:42:26,217 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=136 2024-11-23T05:42:26,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=136, resume processing ppid=135 2024-11-23T05:42:26,219 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=136, ppid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3300 sec 2024-11-23T05:42:26,220 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=135, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=135, table=TestAcidGuarantees in 2.3330 sec 2024-11-23T05:42:26,521 INFO [Thread-1922 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1922. 2024-11-23T05:42:26,521 INFO [Thread-1914 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1914. 2024-11-23T05:42:26,521 INFO [Thread-1916 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-1916. 2024-11-23T05:42:26,521 DEBUG [Thread-1914 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:26,521 DEBUG [Thread-1916 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:26,521 DEBUG [Thread-1922 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:26,521 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:26,521 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:26,521 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:26,521 DEBUG [Thread-1922 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:26,521 DEBUG [Thread-1914 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:26,521 DEBUG [Thread-1916 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:26,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:26,521 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:26,522 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:28,067 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=135 2024-11-23T05:42:28,068 INFO [RPCClient-NioEventLoopGroup-4-10 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 47 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 55 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 70 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 71 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 45 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T05:42:28,068 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1520 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4559 rows 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1519 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4553 rows 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1521 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4562 rows 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1526 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4576 rows 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(402): scanned 1520 2024-11-23T05:42:28,069 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(403): verified 4556 rows 2024-11-23T05:42:28,069 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:42:28,070 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:91) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testScanAtomicity(AcidGuaranteesTestBase.java:112) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:42:28,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:28,070 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:28,070 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:28,072 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T05:42:28,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=137, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:28,078 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T05:42:28,079 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340548079"}]},"ts":"1732340548079"} 2024-11-23T05:42:28,081 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T05:42:28,081 INFO [PEWorker-3 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T05:42:28,082 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=138, ppid=137, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:42:28,083 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, UNASSIGN}] 2024-11-23T05:42:28,084 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, UNASSIGN 2024-11-23T05:42:28,085 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=96da1e6d03a7b375833ea586fe9dc796, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:28,086 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=139, ppid=138, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, UNASSIGN because future has completed 2024-11-23T05:42:28,087 DEBUG [PEWorker-5 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:42:28,087 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:42:28,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T05:42:28,240 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(122): Close 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:28,240 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:42:28,241 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1722): Closing 96da1e6d03a7b375833ea586fe9dc796, disabling compactions & flushes 2024-11-23T05:42:28,241 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:28,241 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:28,241 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. after waiting 0 ms 2024-11-23T05:42:28,241 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:28,241 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(2902): Flushing 96da1e6d03a7b375833ea586fe9dc796 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T05:42:28,249 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/ffa00268ddcf4dc3a931cf584a84ae2b is 50, key is test_row_0/A:col10/1732340545916/Put/seqid=0 2024-11-23T05:42:28,253 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742329_1505 (size=9857) 2024-11-23T05:42:28,397 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T05:42:28,655 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/ffa00268ddcf4dc3a931cf584a84ae2b 2024-11-23T05:42:28,668 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/0245840b0cf3467db17c94f9fe9a7d60 is 50, key is test_row_0/B:col10/1732340545916/Put/seqid=0 2024-11-23T05:42:28,671 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742330_1506 (size=9857) 2024-11-23T05:42:28,706 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T05:42:29,073 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/0245840b0cf3467db17c94f9fe9a7d60 2024-11-23T05:42:29,086 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/a238bc6637034e50b9cc86129f0cd5fa is 50, key is test_row_0/C:col10/1732340545916/Put/seqid=0 2024-11-23T05:42:29,089 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742331_1507 (size=9857) 2024-11-23T05:42:29,216 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T05:42:29,491 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=373 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/a238bc6637034e50b9cc86129f0cd5fa 2024-11-23T05:42:29,502 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/A/ffa00268ddcf4dc3a931cf584a84ae2b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ffa00268ddcf4dc3a931cf584a84ae2b 2024-11-23T05:42:29,506 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ffa00268ddcf4dc3a931cf584a84ae2b, entries=100, sequenceid=373, filesize=9.6 K 2024-11-23T05:42:29,507 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/B/0245840b0cf3467db17c94f9fe9a7d60 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0245840b0cf3467db17c94f9fe9a7d60 2024-11-23T05:42:29,511 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0245840b0cf3467db17c94f9fe9a7d60, entries=100, sequenceid=373, filesize=9.6 K 2024-11-23T05:42:29,512 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/.tmp/C/a238bc6637034e50b9cc86129f0cd5fa as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a238bc6637034e50b9cc86129f0cd5fa 2024-11-23T05:42:29,516 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a238bc6637034e50b9cc86129f0cd5fa, entries=100, sequenceid=373, filesize=9.6 K 2024-11-23T05:42:29,517 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 96da1e6d03a7b375833ea586fe9dc796 in 1275ms, sequenceid=373, compaction requested=true 2024-11-23T05:42:29,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/17676d03070a47c69e7e854e04c270e4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/4651eceddf384d2f9a85805735110ea5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/c922bb5d7cbd422b9d983d07ceffb0b2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e895ace984b94e23aaa49848cbe46eec, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e41dee3c045c46b98277ba03af073eeb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/1f1e158b07094b7bbf41a9d60aa494fc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/20ef81033950492db97c13cddd2c85db, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/045fbff69ee64516982dfa74ae67e564, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/50be77dcf9484badaf1693fbef2b112c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/3327e155e05342aaae9b1748e1c51258, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7b67c24fc8cd4610b2bfa15e55f2c2f8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/82c7ff5d0e134427bf739d6379afccc3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/36e94749e13247fab28701c92f6062ff, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/74b78433e0234547ba1ae7634f2139a8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/f47534ee2ff04597a76947fa3ed1f1f9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/88e8ea6e52234da38010ccaa6ad767c2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/49a448c9ca01406bbcff6fdcd2911c4b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7c3bc185865240938786fa0a68dff44a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5be0ddd76d314483b387a3c9753dbeed, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ce90503f0a9e4a35bc24bc837f35273c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5c4ba999087d4229a6c56820e6ea90bc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/65e26b0a28cb4c59adccb7f8918abb40, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/d0dc1fd00f4444d2849c879cadfd9245, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/0c3fa94486a24f88bb5a7bf99456b9d4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/723fe65c68914ebc97ed74cdd131b1b7] to archive 2024-11-23T05:42:29,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:42:29,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/17676d03070a47c69e7e854e04c270e4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/17676d03070a47c69e7e854e04c270e4 2024-11-23T05:42:29,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/4651eceddf384d2f9a85805735110ea5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/4651eceddf384d2f9a85805735110ea5 2024-11-23T05:42:29,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/c922bb5d7cbd422b9d983d07ceffb0b2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/c922bb5d7cbd422b9d983d07ceffb0b2 2024-11-23T05:42:29,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e895ace984b94e23aaa49848cbe46eec to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e895ace984b94e23aaa49848cbe46eec 2024-11-23T05:42:29,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e41dee3c045c46b98277ba03af073eeb to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/e41dee3c045c46b98277ba03af073eeb 2024-11-23T05:42:29,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/1f1e158b07094b7bbf41a9d60aa494fc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/1f1e158b07094b7bbf41a9d60aa494fc 2024-11-23T05:42:29,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/20ef81033950492db97c13cddd2c85db to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/20ef81033950492db97c13cddd2c85db 2024-11-23T05:42:29,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/045fbff69ee64516982dfa74ae67e564 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/045fbff69ee64516982dfa74ae67e564 2024-11-23T05:42:29,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/50be77dcf9484badaf1693fbef2b112c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/50be77dcf9484badaf1693fbef2b112c 2024-11-23T05:42:29,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/3327e155e05342aaae9b1748e1c51258 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/3327e155e05342aaae9b1748e1c51258 2024-11-23T05:42:29,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7b67c24fc8cd4610b2bfa15e55f2c2f8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7b67c24fc8cd4610b2bfa15e55f2c2f8 2024-11-23T05:42:29,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/82c7ff5d0e134427bf739d6379afccc3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/82c7ff5d0e134427bf739d6379afccc3 2024-11-23T05:42:29,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/36e94749e13247fab28701c92f6062ff to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/36e94749e13247fab28701c92f6062ff 2024-11-23T05:42:29,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/74b78433e0234547ba1ae7634f2139a8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/74b78433e0234547ba1ae7634f2139a8 2024-11-23T05:42:29,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/f47534ee2ff04597a76947fa3ed1f1f9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/f47534ee2ff04597a76947fa3ed1f1f9 2024-11-23T05:42:29,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/88e8ea6e52234da38010ccaa6ad767c2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/88e8ea6e52234da38010ccaa6ad767c2 2024-11-23T05:42:29,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/49a448c9ca01406bbcff6fdcd2911c4b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/49a448c9ca01406bbcff6fdcd2911c4b 2024-11-23T05:42:29,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7c3bc185865240938786fa0a68dff44a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/7c3bc185865240938786fa0a68dff44a 2024-11-23T05:42:29,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5be0ddd76d314483b387a3c9753dbeed to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5be0ddd76d314483b387a3c9753dbeed 2024-11-23T05:42:29,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ce90503f0a9e4a35bc24bc837f35273c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ce90503f0a9e4a35bc24bc837f35273c 2024-11-23T05:42:29,534 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5c4ba999087d4229a6c56820e6ea90bc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/5c4ba999087d4229a6c56820e6ea90bc 2024-11-23T05:42:29,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/65e26b0a28cb4c59adccb7f8918abb40 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/65e26b0a28cb4c59adccb7f8918abb40 2024-11-23T05:42:29,535 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/d0dc1fd00f4444d2849c879cadfd9245 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/d0dc1fd00f4444d2849c879cadfd9245 2024-11-23T05:42:29,536 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/0c3fa94486a24f88bb5a7bf99456b9d4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/0c3fa94486a24f88bb5a7bf99456b9d4 2024-11-23T05:42:29,537 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/723fe65c68914ebc97ed74cdd131b1b7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/723fe65c68914ebc97ed74cdd131b1b7 2024-11-23T05:42:29,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/11240e19f19040fcba2696acdab7c26d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/25ab75c8f9f441e39f39d1df19ff92b4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/10a55d34308d4395b122a18725bfdca2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3604d796da2c40249836dce287aa383e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/99e69acf48ae475084f02f3d8bd77549, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/4b2dcbe18dd44af3af55b16bd1435222, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/96552792406746f69a96bb974e5560ba, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/45fd6f624a7942978c7c02a0e2bb135e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/66dbcc6ef88c4ae393457fd827f35fe0, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/69c53878def944279863ef32ca9e6ced, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6c74a1c15116456e918b35339ea2572f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/1a74ad8cec844bd89feb28f274800e4e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6eb1a433e93446eab19ca15dfc1559f8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0f62e1a7dd254283b662a814b8f67e02, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/40731f0c3cdc46cb89bcebdf3371c2ac, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6bc53c5fe0bb4844805b148809357bba, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5361dccdc8f444a9923b846bc6bde0b4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3ecfe51139e4469e8c915bd36d0a4a10, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/c67ce7959b2a44d0921b9c485e9c0cb4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/962790488dd840e3a735b1389589dd07, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5d986d622a4f47079541bc68a77b47be, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/91a3da931e044a1eb5c0685e9905bf49, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/a712c94c09784945842da96358501e6f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/2cae2ca63bcb4cfba6480d3bff68b4bb, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6af7973235a5468b9ac85c4f51c738c5] to archive 2024-11-23T05:42:29,538 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:42:29,539 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/11240e19f19040fcba2696acdab7c26d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/11240e19f19040fcba2696acdab7c26d 2024-11-23T05:42:29,540 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/25ab75c8f9f441e39f39d1df19ff92b4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/25ab75c8f9f441e39f39d1df19ff92b4 2024-11-23T05:42:29,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/10a55d34308d4395b122a18725bfdca2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/10a55d34308d4395b122a18725bfdca2 2024-11-23T05:42:29,541 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3604d796da2c40249836dce287aa383e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3604d796da2c40249836dce287aa383e 2024-11-23T05:42:29,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/99e69acf48ae475084f02f3d8bd77549 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/99e69acf48ae475084f02f3d8bd77549 2024-11-23T05:42:29,542 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/4b2dcbe18dd44af3af55b16bd1435222 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/4b2dcbe18dd44af3af55b16bd1435222 2024-11-23T05:42:29,543 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/96552792406746f69a96bb974e5560ba to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/96552792406746f69a96bb974e5560ba 2024-11-23T05:42:29,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/45fd6f624a7942978c7c02a0e2bb135e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/45fd6f624a7942978c7c02a0e2bb135e 2024-11-23T05:42:29,544 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/66dbcc6ef88c4ae393457fd827f35fe0 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/66dbcc6ef88c4ae393457fd827f35fe0 2024-11-23T05:42:29,545 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/69c53878def944279863ef32ca9e6ced to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/69c53878def944279863ef32ca9e6ced 2024-11-23T05:42:29,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6c74a1c15116456e918b35339ea2572f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6c74a1c15116456e918b35339ea2572f 2024-11-23T05:42:29,546 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/1a74ad8cec844bd89feb28f274800e4e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/1a74ad8cec844bd89feb28f274800e4e 2024-11-23T05:42:29,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6eb1a433e93446eab19ca15dfc1559f8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6eb1a433e93446eab19ca15dfc1559f8 2024-11-23T05:42:29,547 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0f62e1a7dd254283b662a814b8f67e02 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0f62e1a7dd254283b662a814b8f67e02 2024-11-23T05:42:29,548 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/40731f0c3cdc46cb89bcebdf3371c2ac to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/40731f0c3cdc46cb89bcebdf3371c2ac 2024-11-23T05:42:29,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6bc53c5fe0bb4844805b148809357bba to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6bc53c5fe0bb4844805b148809357bba 2024-11-23T05:42:29,549 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5361dccdc8f444a9923b846bc6bde0b4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5361dccdc8f444a9923b846bc6bde0b4 2024-11-23T05:42:29,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3ecfe51139e4469e8c915bd36d0a4a10 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/3ecfe51139e4469e8c915bd36d0a4a10 2024-11-23T05:42:29,550 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/c67ce7959b2a44d0921b9c485e9c0cb4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/c67ce7959b2a44d0921b9c485e9c0cb4 2024-11-23T05:42:29,551 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/962790488dd840e3a735b1389589dd07 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/962790488dd840e3a735b1389589dd07 2024-11-23T05:42:29,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5d986d622a4f47079541bc68a77b47be to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/5d986d622a4f47079541bc68a77b47be 2024-11-23T05:42:29,552 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/91a3da931e044a1eb5c0685e9905bf49 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/91a3da931e044a1eb5c0685e9905bf49 2024-11-23T05:42:29,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/a712c94c09784945842da96358501e6f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/a712c94c09784945842da96358501e6f 2024-11-23T05:42:29,553 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/2cae2ca63bcb4cfba6480d3bff68b4bb to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/2cae2ca63bcb4cfba6480d3bff68b4bb 2024-11-23T05:42:29,554 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6af7973235a5468b9ac85c4f51c738c5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/6af7973235a5468b9ac85c4f51c738c5 2024-11-23T05:42:29,555 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e0753e6f5e714c9bbc3e9708f1e19096, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cd86a9ab39214fcd810b030f998fd77f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/382dce48b7ce45fface8da443d7e4f9f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e125235eef34466f92f90544b46f5f18, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d0495ca3bc734ffbbf5ffbc678b16bc1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/338cc5bc57814f1ba91edad5ed370c48, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d037aaf5c1024de89e5c8bdc414fe58d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/6a684aa0f8a94deea669c42d2dde0022, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4ae200dbbab46e59f1c16e4a7ee66fd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/0278dedfe40749fa9c5acbe99210dd27, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/ab833def08394ca699184ab7364f5bef, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a23a0287a09e424ca2ae29a743b744d8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/dded319cbc1f474488a597058c335255, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4b5786e738043bb817218de4a879f64, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/8790993497374499b2bd03c491473dc5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e39894d757ef48179b86bc56f3cb617f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/4bac736adb524594934bace63b95edd6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/2ffc7397712c4b9cabdf70eb0207a048, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/923b4055a4f244ee8b536ede5faa074c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cc6b4a3c840640a1972cf7eec4bce8e9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/907db295ca5749aa965792fdab0ee094, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/22a2c25efdb0496d9ad4d8afe183b7c5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/287751cb38104349aa289721dce447d9, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/3608618aba03484e82aa6963da8c243a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/7730aac076c24e23a02f7f137ed30945] to archive 2024-11-23T05:42:29,556 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:42:29,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e0753e6f5e714c9bbc3e9708f1e19096 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e0753e6f5e714c9bbc3e9708f1e19096 2024-11-23T05:42:29,557 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cd86a9ab39214fcd810b030f998fd77f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cd86a9ab39214fcd810b030f998fd77f 2024-11-23T05:42:29,558 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/382dce48b7ce45fface8da443d7e4f9f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/382dce48b7ce45fface8da443d7e4f9f 2024-11-23T05:42:29,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e125235eef34466f92f90544b46f5f18 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e125235eef34466f92f90544b46f5f18 2024-11-23T05:42:29,559 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d0495ca3bc734ffbbf5ffbc678b16bc1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d0495ca3bc734ffbbf5ffbc678b16bc1 2024-11-23T05:42:29,560 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/338cc5bc57814f1ba91edad5ed370c48 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/338cc5bc57814f1ba91edad5ed370c48 2024-11-23T05:42:29,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d037aaf5c1024de89e5c8bdc414fe58d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d037aaf5c1024de89e5c8bdc414fe58d 2024-11-23T05:42:29,561 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/6a684aa0f8a94deea669c42d2dde0022 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/6a684aa0f8a94deea669c42d2dde0022 2024-11-23T05:42:29,562 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4ae200dbbab46e59f1c16e4a7ee66fd to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4ae200dbbab46e59f1c16e4a7ee66fd 2024-11-23T05:42:29,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/0278dedfe40749fa9c5acbe99210dd27 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/0278dedfe40749fa9c5acbe99210dd27 2024-11-23T05:42:29,563 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/ab833def08394ca699184ab7364f5bef to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/ab833def08394ca699184ab7364f5bef 2024-11-23T05:42:29,564 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a23a0287a09e424ca2ae29a743b744d8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a23a0287a09e424ca2ae29a743b744d8 2024-11-23T05:42:29,565 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/dded319cbc1f474488a597058c335255 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/dded319cbc1f474488a597058c335255 2024-11-23T05:42:29,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4b5786e738043bb817218de4a879f64 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/d4b5786e738043bb817218de4a879f64 2024-11-23T05:42:29,566 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/8790993497374499b2bd03c491473dc5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/8790993497374499b2bd03c491473dc5 2024-11-23T05:42:29,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e39894d757ef48179b86bc56f3cb617f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/e39894d757ef48179b86bc56f3cb617f 2024-11-23T05:42:29,567 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/4bac736adb524594934bace63b95edd6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/4bac736adb524594934bace63b95edd6 2024-11-23T05:42:29,568 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/2ffc7397712c4b9cabdf70eb0207a048 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/2ffc7397712c4b9cabdf70eb0207a048 2024-11-23T05:42:29,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/923b4055a4f244ee8b536ede5faa074c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/923b4055a4f244ee8b536ede5faa074c 2024-11-23T05:42:29,569 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cc6b4a3c840640a1972cf7eec4bce8e9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/cc6b4a3c840640a1972cf7eec4bce8e9 2024-11-23T05:42:29,570 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/907db295ca5749aa965792fdab0ee094 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/907db295ca5749aa965792fdab0ee094 2024-11-23T05:42:29,571 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/22a2c25efdb0496d9ad4d8afe183b7c5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/22a2c25efdb0496d9ad4d8afe183b7c5 2024-11-23T05:42:29,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/287751cb38104349aa289721dce447d9 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/287751cb38104349aa289721dce447d9 2024-11-23T05:42:29,572 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/3608618aba03484e82aa6963da8c243a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/3608618aba03484e82aa6963da8c243a 2024-11-23T05:42:29,573 DEBUG [StoreCloser-TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/7730aac076c24e23a02f7f137ed30945 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/7730aac076c24e23a02f7f137ed30945 2024-11-23T05:42:29,576 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/recovered.edits/376.seqid, newMaxSeqId=376, maxSeqId=1 2024-11-23T05:42:29,577 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. 2024-11-23T05:42:29,577 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] regionserver.HRegion(1676): Region close journal for 96da1e6d03a7b375833ea586fe9dc796: Waiting for close lock at 1732340548241Running coprocessor pre-close hooks at 1732340548241Disabling compacts and flushes for region at 1732340548241Disabling writes for close at 1732340548241Obtaining lock to block concurrent updates at 1732340548241Preparing flush snapshotting stores in 96da1e6d03a7b375833ea586fe9dc796 at 1732340548241Finished memstore snapshotting TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732340548242 (+1 ms)Flushing stores of TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796. at 1732340548243 (+1 ms)Flushing 96da1e6d03a7b375833ea586fe9dc796/A: creating writer at 1732340548243Flushing 96da1e6d03a7b375833ea586fe9dc796/A: appending metadata at 1732340548248 (+5 ms)Flushing 96da1e6d03a7b375833ea586fe9dc796/A: closing flushed file at 1732340548248Flushing 96da1e6d03a7b375833ea586fe9dc796/B: creating writer at 1732340548664 (+416 ms)Flushing 96da1e6d03a7b375833ea586fe9dc796/B: appending metadata at 1732340548668 (+4 ms)Flushing 96da1e6d03a7b375833ea586fe9dc796/B: closing flushed file at 1732340548668Flushing 96da1e6d03a7b375833ea586fe9dc796/C: creating writer at 1732340549083 (+415 ms)Flushing 96da1e6d03a7b375833ea586fe9dc796/C: appending metadata at 1732340549085 (+2 ms)Flushing 96da1e6d03a7b375833ea586fe9dc796/C: closing flushed file at 1732340549085Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@664781e5: reopening flushed file at 1732340549501 (+416 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@3ac69236: reopening flushed file at 1732340549506 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@796970d3: reopening flushed file at 1732340549511 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for 96da1e6d03a7b375833ea586fe9dc796 in 1275ms, sequenceid=373, compaction requested=true at 1732340549517 (+6 ms)Writing region close event to WAL at 1732340549574 (+57 ms)Running coprocessor post-close hooks at 1732340549577 (+3 ms)Closed at 1732340549577 2024-11-23T05:42:29,578 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=140}] handler.UnassignRegionHandler(157): Closed 96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:29,579 INFO [PEWorker-3 {}] assignment.RegionStateStore(223): pid=139 updating hbase:meta row=96da1e6d03a7b375833ea586fe9dc796, regionState=CLOSED 2024-11-23T05:42:29,580 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=140, ppid=139, state=RUNNABLE, hasLock=false; CloseRegionProcedure 96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:42:29,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=140, resume processing ppid=139 2024-11-23T05:42:29,582 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=140, ppid=139, state=SUCCESS, hasLock=false; CloseRegionProcedure 96da1e6d03a7b375833ea586fe9dc796, server=9abfc7c2569e,45719,1732340394120 in 1.4930 sec 2024-11-23T05:42:29,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=139, resume processing ppid=138 2024-11-23T05:42:29,583 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=139, ppid=138, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=96da1e6d03a7b375833ea586fe9dc796, UNASSIGN in 1.4990 sec 2024-11-23T05:42:29,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=138, resume processing ppid=137 2024-11-23T05:42:29,585 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=138, ppid=137, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.5020 sec 2024-11-23T05:42:29,586 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340549586"}]},"ts":"1732340549586"} 2024-11-23T05:42:29,587 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T05:42:29,587 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T05:42:29,588 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=137, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.5150 sec 2024-11-23T05:42:30,227 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=137 2024-11-23T05:42:30,227 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:30,227 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T05:42:30,228 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:30,229 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=141, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:30,230 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T05:42:30,230 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=141, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:30,232 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:30,234 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/recovered.edits] 2024-11-23T05:42:30,238 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/6ee7b5fccab6415b9c998a156dbc30d8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/6ee7b5fccab6415b9c998a156dbc30d8 2024-11-23T05:42:30,240 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/98c73615edee4b798f7e9157490bafe6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/98c73615edee4b798f7e9157490bafe6 2024-11-23T05:42:30,242 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ffa00268ddcf4dc3a931cf584a84ae2b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/A/ffa00268ddcf4dc3a931cf584a84ae2b 2024-11-23T05:42:30,245 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0245840b0cf3467db17c94f9fe9a7d60 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/0245840b0cf3467db17c94f9fe9a7d60 2024-11-23T05:42:30,246 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/571aaab513594547b8f8674512d9a5f4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/571aaab513594547b8f8674512d9a5f4 2024-11-23T05:42:30,248 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/57f8575ed88f490ab6ec53472508ae67 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/B/57f8575ed88f490ab6ec53472508ae67 2024-11-23T05:42:30,251 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/115666f2df094041aa1be25619cb9782 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/115666f2df094041aa1be25619cb9782 2024-11-23T05:42:30,253 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a238bc6637034e50b9cc86129f0cd5fa to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/a238bc6637034e50b9cc86129f0cd5fa 2024-11-23T05:42:30,254 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/c7783b31bd1d4c398f337ac351b934ae to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/C/c7783b31bd1d4c398f337ac351b934ae 2024-11-23T05:42:30,258 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/recovered.edits/376.seqid to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796/recovered.edits/376.seqid 2024-11-23T05:42:30,259 DEBUG [HFileArchiver-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/96da1e6d03a7b375833ea586fe9dc796 2024-11-23T05:42:30,259 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T05:42:30,262 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=141, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:30,264 WARN [PEWorker-3 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T05:42:30,267 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T05:42:30,269 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=141, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:30,269 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T05:42:30,269 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732340550269"}]},"ts":"9223372036854775807"} 2024-11-23T05:42:30,272 INFO [PEWorker-3 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T05:42:30,272 DEBUG [PEWorker-3 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => 96da1e6d03a7b375833ea586fe9dc796, NAME => 'TestAcidGuarantees,,1732340522622.96da1e6d03a7b375833ea586fe9dc796.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T05:42:30,272 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T05:42:30,272 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732340550272"}]},"ts":"9223372036854775807"} 2024-11-23T05:42:30,276 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T05:42:30,277 DEBUG [PEWorker-3 {}] procedure.DeleteTableProcedure(133): Finished pid=141, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:30,278 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=141, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 50 msec 2024-11-23T05:42:30,336 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=141 2024-11-23T05:42:30,337 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T05:42:30,337 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:30,344 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testScanAtomicity Thread=241 (was 243), OpenFileDescriptor=451 (was 459), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=427 (was 445), ProcessCount=11 (was 11), AvailableMemoryMB=8297 (was 8665) 2024-11-23T05:42:30,352 INFO [Time-limited test {}] hbase.ResourceChecker(147): before: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=241, OpenFileDescriptor=451, MaxFileDescriptor=1048576, SystemLoadAverage=427, ProcessCount=11, AvailableMemoryMB=8296 2024-11-23T05:42:30,353 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:42:30,353 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$4(2454): Client=jenkins//172.17.0.2 create 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:42:30,354 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=false; CreateTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:30,355 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_PRE_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_PRE_OPERATION 2024-11-23T05:42:30,355 DEBUG [PEWorker-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:30,355 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(787): Client=jenkins//172.17.0.2 procedure request for creating table: namespace: "default" qualifier: "TestAcidGuarantees" procId is: 142 2024-11-23T05:42:30,356 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T05:42:30,356 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_WRITE_FS_LAYOUT, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_WRITE_FS_LAYOUT 2024-11-23T05:42:30,360 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742332_1508 (size=959) 2024-11-23T05:42:30,466 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T05:42:30,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T05:42:30,766 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(7572): creating {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''}, tableDescriptor='TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, regionDir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d 2024-11-23T05:42:30,775 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742333_1509 (size=53) 2024-11-23T05:42:30,986 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T05:42:31,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:42:31,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1722): Closing f941d6128e412cb98f4108ef00511119, disabling compactions & flushes 2024-11-23T05:42:31,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:31,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:31,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. after waiting 0 ms 2024-11-23T05:42:31,176 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:31,176 INFO [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:31,177 DEBUG [RegionOpenAndInit-TestAcidGuarantees-pool-0 {}] regionserver.HRegion(1676): Region close journal for f941d6128e412cb98f4108ef00511119: Waiting for close lock at 1732340551176Disabling compacts and flushes for region at 1732340551176Disabling writes for close at 1732340551176Writing region close event to WAL at 1732340551176Closed at 1732340551176 2024-11-23T05:42:31,178 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ADD_TO_META, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ADD_TO_META 2024-11-23T05:42:31,178 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":2,"row":"TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.","families":{"info":[{"qualifier":"regioninfo","vlen":52,"tag":[],"timestamp":"1732340551178"},{"qualifier":"state","vlen":6,"tag":[],"timestamp":"1732340551178"}]},"ts":"1732340551178"} 2024-11-23T05:42:31,180 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(832): Added 1 regions to meta. 2024-11-23T05:42:31,180 INFO [PEWorker-1 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_ASSIGN_REGIONS, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_ASSIGN_REGIONS 2024-11-23T05:42:31,181 DEBUG [PEWorker-1 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340551180"}]},"ts":"1732340551180"} 2024-11-23T05:42:31,182 INFO [PEWorker-1 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLING in hbase:meta 2024-11-23T05:42:31,182 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, ASSIGN}] 2024-11-23T05:42:31,183 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, ASSIGN 2024-11-23T05:42:31,184 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, ASSIGN; state=OFFLINE, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=false 2024-11-23T05:42:31,335 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:31,336 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=143, ppid=142, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, ASSIGN because future has completed 2024-11-23T05:42:31,337 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:42:31,494 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:31,495 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7752): Opening region: {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:42:31,495 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,495 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:42:31,496 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7794): checking encryption for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,496 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(7797): checking classloading for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,496 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T05:42:31,498 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,499 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f941d6128e412cb98f4108ef00511119 columnFamilyName A 2024-11-23T05:42:31,499 DEBUG [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:31,500 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(327): Store=f941d6128e412cb98f4108ef00511119/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:31,500 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,501 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f941d6128e412cb98f4108ef00511119 columnFamilyName B 2024-11-23T05:42:31,501 DEBUG [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:31,502 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(327): Store=f941d6128e412cb98f4108ef00511119/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:31,502 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,503 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f941d6128e412cb98f4108ef00511119 columnFamilyName C 2024-11-23T05:42:31,503 DEBUG [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:31,503 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(327): Store=f941d6128e412cb98f4108ef00511119/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:31,503 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1038): replaying wal for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,504 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,504 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,505 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1048): stopping wal replay for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,505 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1060): Cleaning up temporary data for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,506 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:42:31,507 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1093): writing seq id for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,508 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/recovered.edits/1.seqid, newMaxSeqId=1, maxSeqId=-1 2024-11-23T05:42:31,509 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1114): Opened f941d6128e412cb98f4108ef00511119; next sequenceid=2; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=61719676, jitterRate=-0.08030515909194946}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:42:31,509 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:31,509 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegion(1006): Region open journal for f941d6128e412cb98f4108ef00511119: Running coprocessor pre-open hook at 1732340551496Writing region info on filesystem at 1732340551496Initializing all the Stores at 1732340551497 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340551497Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340551498 (+1 ms)Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340551498Cleaning up temporary data from old regions at 1732340551505 (+7 ms)Running coprocessor post-open hooks at 1732340551509 (+4 ms)Region opened successfully at 1732340551509 2024-11-23T05:42:31,510 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., pid=144, masterSystemTime=1732340551488 2024-11-23T05:42:31,511 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:31,511 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=144}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:31,512 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=143 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=OPEN, openSeqNum=2, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:31,513 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=144, ppid=143, state=RUNNABLE, hasLock=false; OpenRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:42:31,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=144, resume processing ppid=143 2024-11-23T05:42:31,515 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=144, ppid=143, state=SUCCESS, hasLock=false; OpenRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 in 176 msec 2024-11-23T05:42:31,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=143, resume processing ppid=142 2024-11-23T05:42:31,517 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=143, ppid=142, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, ASSIGN in 333 msec 2024-11-23T05:42:31,517 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_UPDATE_DESC_CACHE, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_UPDATE_DESC_CACHE 2024-11-23T05:42:31,518 DEBUG [PEWorker-3 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340551517"}]},"ts":"1732340551517"} 2024-11-23T05:42:31,519 INFO [PEWorker-3 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=ENABLED in hbase:meta 2024-11-23T05:42:31,520 INFO [PEWorker-3 {}] procedure.CreateTableProcedure(89): pid=142, state=RUNNABLE:CREATE_TABLE_POST_OPERATION, hasLock=true; CreateTableProcedure table=TestAcidGuarantees execute state=CREATE_TABLE_POST_OPERATION 2024-11-23T05:42:31,521 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=142, state=SUCCESS, hasLock=false; CreateTableProcedure table=TestAcidGuarantees in 1.1670 sec 2024-11-23T05:42:32,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=142 2024-11-23T05:42:32,507 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: CREATE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:32,509 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5185cce8, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:32,509 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:32,510 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:32,512 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:32,512 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1c6a5565, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:32,513 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:32,514 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:32,516 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33138, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:32,518 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4841ebc4, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:32,518 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:32,519 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:32,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53976, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:32,521 DEBUG [Time-limited test {}] client.ConnectionUtils(547): Start fetching master stub from registry 2024-11-23T05:42:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncConnectionImpl(321): The fetched master address is 9abfc7c2569e,40189,1732340393249 2024-11-23T05:42:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched master stub is org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProtos$MasterService$Stub@40e4be62 2024-11-23T05:42:32,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=MasterService, sasl=false 2024-11-23T05:42:32,522 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33144, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=MasterService 2024-11-23T05:42:32,523 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] util.TableDescriptorChecker(321): MEMSTORE_FLUSHSIZE for table descriptor or "hbase.hregion.memstore.flush.size" (131072) is too small, which might cause very frequent flushing. 2024-11-23T05:42:32,524 INFO [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.HMaster$14(2863): Client=jenkins//172.17.0.2 modify table TestAcidGuarantees from 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} to 'TestAcidGuarantees', {TABLE_ATTRIBUTES => {METADATA => {'hbase.hregion.compacting.memstore.type' => 'NONE', 'hbase.store.file-tracker.impl' => 'DEFAULT'}}}, {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'}, {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} 2024-11-23T05:42:32,525 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=145, state=RUNNABLE:MODIFY_TABLE_PREPARE, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:32,534 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742334_1510 (size=995) 2024-11-23T05:42:32,935 DEBUG [PEWorker-5 {}] util.FSTableDescriptors(591): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000001.959 2024-11-23T05:42:32,936 INFO [PEWorker-5 {}] util.FSTableDescriptors(361): Updated tableinfo=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/.tabledesc/.tableinfo.0000000002.995 2024-11-23T05:42:32,938 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=146, ppid=145, state=RUNNABLE:REOPEN_TABLE_REGIONS_GET_REGIONS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:42:32,940 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, REOPEN/MOVE}] 2024-11-23T05:42:32,941 INFO [PEWorker-4 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, REOPEN/MOVE 2024-11-23T05:42:32,942 INFO [PEWorker-4 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:32,943 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, REOPEN/MOVE because future has completed 2024-11-23T05:42:32,943 DEBUG [PEWorker-2 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:42:32,943 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:42:33,096 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(122): Close f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,096 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:42:33,096 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1722): Closing f941d6128e412cb98f4108ef00511119, disabling compactions & flushes 2024-11-23T05:42:33,096 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,096 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,096 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. after waiting 0 ms 2024-11-23T05:42:33,096 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,100 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/recovered.edits/4.seqid, newMaxSeqId=4, maxSeqId=1 2024-11-23T05:42:33,100 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,101 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegion(1676): Region close journal for f941d6128e412cb98f4108ef00511119: Waiting for close lock at 1732340553096Running coprocessor pre-close hooks at 1732340553096Disabling compacts and flushes for region at 1732340553096Disabling writes for close at 1732340553096Writing region close event to WAL at 1732340553097 (+1 ms)Running coprocessor post-close hooks at 1732340553100 (+3 ms)Closed at 1732340553100 2024-11-23T05:42:33,101 WARN [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] regionserver.HRegionServer(3299): Not adding moved region record: f941d6128e412cb98f4108ef00511119 to self. 2024-11-23T05:42:33,103 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=148}] handler.UnassignRegionHandler(157): Closed f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,103 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=CLOSED 2024-11-23T05:42:33,104 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=148, ppid=147, state=RUNNABLE, hasLock=false; CloseRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:42:33,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=148, resume processing ppid=147 2024-11-23T05:42:33,106 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=148, ppid=147, state=SUCCESS, hasLock=false; CloseRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 in 162 msec 2024-11-23T05:42:33,107 INFO [PEWorker-4 {}] assignment.TransitRegionStateProcedure(269): Starting pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_GET_ASSIGN_CANDIDATE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, REOPEN/MOVE; state=CLOSED, location=9abfc7c2569e,45719,1732340394120; forceNewPlan=false, retain=true 2024-11-23T05:42:33,257 INFO [PEWorker-2 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=OPENING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,259 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=147, ppid=146, state=RUNNABLE:REGION_STATE_TRANSITION_OPEN, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, REOPEN/MOVE because future has completed 2024-11-23T05:42:33,259 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:42:33,417 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(132): Open TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,417 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7752): Opening region: {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} 2024-11-23T05:42:33,418 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.MetricsRegionSourceImpl(79): Creating new MetricsRegionSourceImpl for table TestAcidGuarantees f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,418 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(898): Instantiated TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.; StoreHotnessProtector, parallelPutToStoreThreadLimit=0 ; minColumnNum=100 ; preparePutThreadLimit=0 ; hotProtect now disable 2024-11-23T05:42:33,418 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7794): checking encryption for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,418 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(7797): checking classloading for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,420 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family A of region f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,421 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f941d6128e412cb98f4108ef00511119 columnFamilyName A 2024-11-23T05:42:33,423 DEBUG [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:33,423 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(327): Store=f941d6128e412cb98f4108ef00511119/A, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:33,424 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family B of region f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,424 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f941d6128e412cb98f4108ef00511119 columnFamilyName B 2024-11-23T05:42:33,424 DEBUG [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:33,425 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(327): Store=f941d6128e412cb98f4108ef00511119/B, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:33,425 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(400): Created cacheConfig: cacheDataOnRead=true, cacheDataOnWrite=false, cacheIndexesOnWrite=false, cacheBloomsOnWrite=false, cacheEvictOnClose=false, cacheDataCompressed=false, prefetchOnOpen=false, for column family C of region f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,426 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] compactions.CompactionConfiguration(183): size [minCompactSize:128 MB, maxCompactSize:8.00 EB, offPeakMaxCompactSize:8.00 EB); files [minFilesToCompact:3, maxFilesToCompact:10); ratio 1.200000; off-peak ratio 5.000000; throttle point 2684354560; major period 604800000, major jitter 0.500000, min locality to compact 0.000000; tiered compaction: max_age 9223372036854775807, incoming window min 6, compaction policy for tiered window org.apache.hadoop.hbase.regionserver.compactions.ExploringCompactionPolicy, single output for minor true, compaction window factory org.apache.hadoop.hbase.regionserver.compactions.ExponentialCompactionWindowFactory, region f941d6128e412cb98f4108ef00511119 columnFamilyName C 2024-11-23T05:42:33,426 DEBUG [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:33,426 INFO [StoreOpener-f941d6128e412cb98f4108ef00511119-1 {}] regionserver.HStore(327): Store=f941d6128e412cb98f4108ef00511119/C, memstore type=DefaultMemStore, storagePolicy=NONE, verifyBulkLoads=false, parallelPutCountPrintThreshold=50, encoding=NONE, compression=NONE 2024-11-23T05:42:33,426 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1038): replaying wal for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,427 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,428 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(5546): Found 0 recovered edits file(s) under hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,429 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1048): stopping wal replay for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,429 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1060): Cleaning up temporary data for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,430 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.FlushLargeStoresPolicy(65): No hbase.hregion.percolumnfamilyflush.size.lower.bound set in table TestAcidGuarantees descriptor;using region.getMemStoreFlushHeapSize/# of families (16.0 M)) instead. 2024-11-23T05:42:33,431 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1093): writing seq id for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,432 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1114): Opened f941d6128e412cb98f4108ef00511119; next sequenceid=5; ConstantSizeRegionSplitPolicy{desiredMaxFileSize=62476991, jitterRate=-0.06902028620243073}, FlushLargeStoresPolicy{flushSizeLowerBound=16777216} 2024-11-23T05:42:33,432 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1122): Running coprocessor post-open hooks for f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,433 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegion(1006): Region open journal for f941d6128e412cb98f4108ef00511119: Running coprocessor pre-open hook at 1732340553419Writing region info on filesystem at 1732340553419Initializing all the Stores at 1732340553420 (+1 ms)Instantiating store for column family {NAME => 'A', INDEX_BLOCK_ENCODING => 'NONE', MOB_THRESHOLD => '4', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', IS_MOB => 'true', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340553420Instantiating store for column family {NAME => 'B', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340553420Instantiating store for column family {NAME => 'C', INDEX_BLOCK_ENCODING => 'NONE', VERSIONS => '1', KEEP_DELETED_CELLS => 'FALSE', DATA_BLOCK_ENCODING => 'NONE', TTL => 'FOREVER', MIN_VERSIONS => '0', REPLICATION_SCOPE => '0', BLOOMFILTER => 'ROW', IN_MEMORY => 'false', COMPRESSION => 'NONE', BLOCKCACHE => 'true', BLOCKSIZE => '65536 B (64KB)'} at 1732340553420Cleaning up temporary data from old regions at 1732340553429 (+9 ms)Running coprocessor post-open hooks at 1732340553432 (+3 ms)Region opened successfully at 1732340553433 (+1 ms) 2024-11-23T05:42:33,434 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2236): Post open deploy tasks for TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., pid=149, masterSystemTime=1732340553412 2024-11-23T05:42:33,436 DEBUG [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] regionserver.HRegionServer(2266): Finished post open deploy task for TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,436 INFO [RS_OPEN_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_OPEN_REGION, pid=149}] handler.AssignRegionHandler(153): Opened TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,437 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=147 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=OPEN, openSeqNum=5, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,438 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=149, ppid=147, state=RUNNABLE, hasLock=false; OpenRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:42:33,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=149, resume processing ppid=147 2024-11-23T05:42:33,443 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=149, ppid=147, state=SUCCESS, hasLock=false; OpenRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 in 181 msec 2024-11-23T05:42:33,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=147, resume processing ppid=146 2024-11-23T05:42:33,446 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=147, ppid=146, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, REOPEN/MOVE in 503 msec 2024-11-23T05:42:33,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=146, resume processing ppid=145 2024-11-23T05:42:33,450 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=146, ppid=145, state=SUCCESS, hasLock=false; ReopenTableRegionsProcedure table=TestAcidGuarantees in 509 msec 2024-11-23T05:42:33,453 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=145, state=SUCCESS, hasLock=false; ModifyTableProcedure table=TestAcidGuarantees in 927 msec 2024-11-23T05:42:33,454 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=145 2024-11-23T05:42:33,454 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: MODIFY_COLUMN_FAMILY, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:33,455 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@64082ee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,455 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,455 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,456 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,456 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7a5669e3, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,457 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,458 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33154, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,459 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6ee00338, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,459 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2ea59a4a, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,459 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,460 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,460 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,460 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@7aafac0c, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,461 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,462 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33180, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,463 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1bf51a28, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,463 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@54b93a72, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,464 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,464 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,464 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@672b9e15, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,465 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,466 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,466 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33196, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,467 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3bacbd7b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,467 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2948979e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,467 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,467 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,468 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@1591d792, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,468 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,469 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,469 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33216, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,470 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@4ed38e1b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,470 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6e715d8d, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,470 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,470 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,471 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@5f0e060e, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,471 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,472 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33232, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,472 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@3781dc76, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,473 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@31b8df26, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,473 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,473 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,473 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,473 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2f9003c6, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,474 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,474 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33252, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,475 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@2769ef94, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,475 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@8f9db7f, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,475 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,475 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,476 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@295b9944, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,476 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,477 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33262, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,477 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@40cc7dee, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,478 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@c89c532, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,478 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,478 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,479 DEBUG [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@606ea292, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,479 DEBUG [RPCClient-NioEventLoopGroup-4-11 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,480 INFO [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33270, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,481 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@fe41126, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,482 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@150c6132, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,482 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,482 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,482 DEBUG [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@14467ded, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,483 DEBUG [RPCClient-NioEventLoopGroup-4-13 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,484 INFO [HMaster-EventLoopGroup-1-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33292, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,484 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@37bb5566, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,485 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@66be6e98, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,485 DEBUG [Time-limited test {}] client.ClusterIdFetcher(90): Going to request 9abfc7c2569e,40189,-1 for getting cluster id 2024-11-23T05:42:33,485 DEBUG [Time-limited test {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ConnectionRegistryService, sasl=false 2024-11-23T05:42:33,486 DEBUG [HMaster-EventLoopGroup-1-3 {}] ipc.ServerRpcConnection(714): Response connection registry, clusterId = '1659c1c0-0bd8-44bc-ada4-34dd97dbfe67' 2024-11-23T05:42:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(424): process preamble call response with response type GetConnectionRegistryResponse 2024-11-23T05:42:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ClusterIdFetcher$1(103): Got connection registry info: cluster_id: "1659c1c0-0bd8-44bc-ada4-34dd97dbfe67" 2024-11-23T05:42:33,486 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@6357e9d2, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] client.ConnectionRegistryRpcStubHolder(93): Going to use new servers to create stubs: [9abfc7c2569e,40189,-1] 2024-11-23T05:42:33,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientMetaService, sasl=false 2024-11-23T05:42:33,487 DEBUG [RPCClient-NioEventLoopGroup-4-15 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:33,488 INFO [HMaster-EventLoopGroup-1-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:33306, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientMetaService 2024-11-23T05:42:33,488 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.AbstractRpcClient(198): Codec=org.apache.hadoop.hbase.codec.KeyValueCodec@523c57b, compressor=null, tcpKeepAlive=true, tcpNoDelay=true, connectTO=10000, readTO=20000, writeTO=60000, minIdleTimeBeforeClose=120000, maxRetries=0, fallbackAllowed=true, bind address=null 2024-11-23T05:42:33,489 DEBUG [Thread-2308 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,489 DEBUG [Thread-2293 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,489 DEBUG [Thread-2302 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,489 DEBUG [Thread-2291 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,490 DEBUG [Thread-2299 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,490 DEBUG [Thread-2306 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,490 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-14 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,491 DEBUG [Thread-2297 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,491 DEBUG [RPCClient-NioEventLoopGroup-4-12 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,491 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53990, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:53998, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,492 DEBUG [Thread-2310 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54012, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54026, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,492 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54042, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,492 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54016, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,493 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54046, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-16 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,493 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,494 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-2 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54054, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,494 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,495 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,496 DEBUG [RPCClient-NioEventLoopGroup-4-6 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,497 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_0', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,498 DEBUG [RPCClient-NioEventLoopGroup-4-8 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,499 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_2', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,499 DEBUG [Thread-2304 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,500 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:33,500 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees 2024-11-23T05:42:33,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T05:42:33,502 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:33,503 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-1 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54068, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,504 DEBUG [RPCClient-NioEventLoopGroup-4-9 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,504 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=150, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:33,504 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=151, ppid=150, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:33,506 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:33,506 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:42:33,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,513 DEBUG [Thread-2295 {}] client.ConnectionUtils(547): Start fetching meta region location from registry 2024-11-23T05:42:33,514 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] client.ConnectionUtils(555): The fetched meta region location is [region=hbase:meta,,1.1588230740, hostname=9abfc7c2569e,45719,1732340394120, seqNum=-1] 2024-11-23T05:42:33,515 DEBUG [RPCClient-NioEventLoopGroup-4-2 {}] ipc.RpcConnection(159): Using SIMPLE authentication for service=ClientService, sasl=false 2024-11-23T05:42:33,520 INFO [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] ipc.ServerRpcConnection(484): Connection from 172.17.0.2:54076, version=4.0.0-alpha-1-SNAPSHOT, sasl=false, ugi=jenkins (auth:SIMPLE), service=ClientService 2024-11-23T05:42:33,521 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncNonMetaRegionLocator(310): The fetched location of 'TestAcidGuarantees', row='test_row_1', locateType=CURRENT is [region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5] 2024-11-23T05:42:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 2 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340613528, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,534 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340613529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,534 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,534 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340613530, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340613529, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,535 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,535 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 7 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340613531, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,535 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,536 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112388d41a2421ea4234b2d50ba0ed7ad95b_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340553501/Put/seqid=0 2024-11-23T05:42:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,536 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,551 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742335_1511 (size=17034) 2024-11-23T05:42:33,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T05:42:33,637 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 3 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340613636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,647 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340613646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340613647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 8 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340613647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,648 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,648 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340613647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,656 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T05:42:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,657 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,657 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,657 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,809 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T05:42:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,809 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:33,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,810 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,810 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,816 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T05:42:33,848 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,848 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 4 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340613846, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,848 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340613856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,858 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340613856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,859 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340613857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 9 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340613857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:33,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:33,952 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:33,955 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112388d41a2421ea4234b2d50ba0ed7ad95b_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112388d41a2421ea4234b2d50ba0ed7ad95b_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:33,955 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0d1ad5e8235b4a94a5b5fdd0d79776b7, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:33,956 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0d1ad5e8235b4a94a5b5fdd0d79776b7 is 175, key is test_row_0/A:col10/1732340553501/Put/seqid=0 2024-11-23T05:42:33,962 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T05:42:33,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:33,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:33,962 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] handler.RSProcedureHandler(58): pid=151 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=151 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,963 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=151 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:33,967 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742336_1512 (size=48139) 2024-11-23T05:42:33,971 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=16, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0d1ad5e8235b4a94a5b5fdd0d79776b7 2024-11-23T05:42:33,992 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/15aa5550c1164693802b7959df6794f3 is 50, key is test_row_0/B:col10/1732340553501/Put/seqid=0 2024-11-23T05:42:33,999 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742337_1513 (size=12001) 2024-11-23T05:42:34,000 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/15aa5550c1164693802b7959df6794f3 2024-11-23T05:42:34,023 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/cc46b4b573b04c8d9a907d5bc06e25e1 is 50, key is test_row_0/C:col10/1732340553501/Put/seqid=0 2024-11-23T05:42:34,026 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742338_1514 (size=12001) 2024-11-23T05:42:34,030 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=16 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/cc46b4b573b04c8d9a907d5bc06e25e1 2024-11-23T05:42:34,033 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0d1ad5e8235b4a94a5b5fdd0d79776b7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d1ad5e8235b4a94a5b5fdd0d79776b7 2024-11-23T05:42:34,037 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d1ad5e8235b4a94a5b5fdd0d79776b7, entries=250, sequenceid=16, filesize=47.0 K 2024-11-23T05:42:34,037 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/15aa5550c1164693802b7959df6794f3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/15aa5550c1164693802b7959df6794f3 2024-11-23T05:42:34,040 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/15aa5550c1164693802b7959df6794f3, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T05:42:34,041 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/cc46b4b573b04c8d9a907d5bc06e25e1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cc46b4b573b04c8d9a907d5bc06e25e1 2024-11-23T05:42:34,044 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cc46b4b573b04c8d9a907d5bc06e25e1, entries=150, sequenceid=16, filesize=11.7 K 2024-11-23T05:42:34,045 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f941d6128e412cb98f4108ef00511119 in 539ms, sequenceid=16, compaction requested=false 2024-11-23T05:42:34,045 DEBUG [MemStoreFlusher.0 {}] regionserver.MetricsTableSourceImpl(133): Creating new MetricsTableSourceImpl for table 'TestAcidGuarantees' 2024-11-23T05:42:34,046 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:34,115 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=151 2024-11-23T05:42:34,115 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:34,115 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:42:34,123 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123262b9aac6085424b95020935ed16321d_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340553530/Put/seqid=0 2024-11-23T05:42:34,126 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T05:42:34,128 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742339_1515 (size=12154) 2024-11-23T05:42:34,128 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:34,132 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123262b9aac6085424b95020935ed16321d_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123262b9aac6085424b95020935ed16321d_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:34,132 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3a91b1c932da40f9ae524240ac8f4835, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:34,133 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3a91b1c932da40f9ae524240ac8f4835 is 175, key is test_row_0/A:col10/1732340553530/Put/seqid=0 2024-11-23T05:42:34,143 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742340_1516 (size=30955) 2024-11-23T05:42:34,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:34,158 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:34,173 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,173 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340614169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340614169, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,174 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340614170, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,174 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,174 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340614171, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,175 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,175 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 10 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340614172, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,175 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340614276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,278 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340614276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,278 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,278 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,279 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340614276, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,279 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340614286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,287 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 11 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340614286, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,287 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,288 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,489 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340614486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,489 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340614486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,490 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340614486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,490 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,498 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340614496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,499 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,499 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 12 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340614496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,499 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,543 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=40, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3a91b1c932da40f9ae524240ac8f4835 2024-11-23T05:42:34,550 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/6bb3ae26edc948f7bcd4aded0bb7a0ab is 50, key is test_row_0/B:col10/1732340553530/Put/seqid=0 2024-11-23T05:42:34,556 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742341_1517 (size=12001) 2024-11-23T05:42:34,636 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T05:42:34,798 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340614796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340614796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,799 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,799 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340614796, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,799 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340614805, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,808 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,808 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 13 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340614806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:34,808 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:34,956 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/6bb3ae26edc948f7bcd4aded0bb7a0ab 2024-11-23T05:42:34,962 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/aef7ef4925484a0787ab6f1d4b9e5bf5 is 50, key is test_row_0/C:col10/1732340553530/Put/seqid=0 2024-11-23T05:42:34,964 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742342_1518 (size=12001) 2024-11-23T05:42:35,099 WARN [HBase-Metrics2-1 {}] impl.MetricsConfig(138): Cannot locate configuration: tried hadoop-metrics2-hbase.properties,hadoop-metrics2.properties 2024-11-23T05:42:35,308 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:35,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340615306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,308 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:35,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340615306, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:35,309 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340615307, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,309 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:35,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 15 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340615316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,318 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:35,318 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 14 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340615316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:35,318 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:35,365 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=40 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/aef7ef4925484a0787ab6f1d4b9e5bf5 2024-11-23T05:42:35,368 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3a91b1c932da40f9ae524240ac8f4835 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3a91b1c932da40f9ae524240ac8f4835 2024-11-23T05:42:35,371 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3a91b1c932da40f9ae524240ac8f4835, entries=150, sequenceid=40, filesize=30.2 K 2024-11-23T05:42:35,371 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/6bb3ae26edc948f7bcd4aded0bb7a0ab as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/6bb3ae26edc948f7bcd4aded0bb7a0ab 2024-11-23T05:42:35,374 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/6bb3ae26edc948f7bcd4aded0bb7a0ab, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T05:42:35,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/aef7ef4925484a0787ab6f1d4b9e5bf5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/aef7ef4925484a0787ab6f1d4b9e5bf5 2024-11-23T05:42:35,377 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/aef7ef4925484a0787ab6f1d4b9e5bf5, entries=150, sequenceid=40, filesize=11.7 K 2024-11-23T05:42:35,378 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f941d6128e412cb98f4108ef00511119 in 1263ms, sequenceid=40, compaction requested=false 2024-11-23T05:42:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:35,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=151}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=151 2024-11-23T05:42:35,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=151 2024-11-23T05:42:35,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=151, resume processing ppid=150 2024-11-23T05:42:35,381 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=151, ppid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.8750 sec 2024-11-23T05:42:35,382 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=150, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=150, table=TestAcidGuarantees in 1.8810 sec 2024-11-23T05:42:35,646 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=150 2024-11-23T05:42:35,646 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:35,647 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees 2024-11-23T05:42:35,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T05:42:35,648 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:35,649 INFO [PEWorker-2 {}] procedure.FlushTableProcedure(91): pid=152, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:35,649 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=153, ppid=152, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:35,756 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T05:42:35,801 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=153 2024-11-23T05:42:35,801 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:35,801 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=60.38 KB heapSize=158.95 KB 2024-11-23T05:42:35,806 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230ff0447d3c204c8d884516df28841ce7_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340554169/Put/seqid=0 2024-11-23T05:42:35,809 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742343_1519 (size=12154) 2024-11-23T05:42:35,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T05:42:36,209 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:36,212 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411230ff0447d3c204c8d884516df28841ce7_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230ff0447d3c204c8d884516df28841ce7_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3ae69c16b7c8418da873e6a32aa06b91, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:36,213 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3ae69c16b7c8418da873e6a32aa06b91 is 175, key is test_row_0/A:col10/1732340554169/Put/seqid=0 2024-11-23T05:42:36,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742344_1520 (size=30955) 2024-11-23T05:42:36,276 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T05:42:36,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:36,317 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:36,346 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 16 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340616342, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340616343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,347 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,347 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340616343, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,347 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340616345, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,350 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,350 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,350 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340616346, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,351 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 17 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340616456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,458 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,458 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340616456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 20 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340616456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340616456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,459 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340616456, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,459 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,616 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=52, memsize=20.1 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3ae69c16b7c8418da873e6a32aa06b91 2024-11-23T05:42:36,621 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/805e1c6ea2ea499a97b0d8538d5b72ba is 50, key is test_row_0/B:col10/1732340554169/Put/seqid=0 2024-11-23T05:42:36,624 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742345_1521 (size=12001) 2024-11-23T05:42:36,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 18 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340616666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,668 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,668 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340616666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,668 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,668 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 21 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340616666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340616666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,669 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,669 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,669 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340616666, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,670 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T05:42:36,976 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,977 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 19 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340616976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,977 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340616976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340616976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,979 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,979 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340616976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,979 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:36,980 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340616976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:36,980 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,024 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/805e1c6ea2ea499a97b0d8538d5b72ba 2024-11-23T05:42:37,029 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/5c10d96ebe78426c875245f83bca34e5 is 50, key is test_row_0/C:col10/1732340554169/Put/seqid=0 2024-11-23T05:42:37,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742346_1522 (size=12001) 2024-11-23T05:42:37,433 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=20.13 KB at sequenceid=52 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/5c10d96ebe78426c875245f83bca34e5 2024-11-23T05:42:37,437 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/3ae69c16b7c8418da873e6a32aa06b91 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3ae69c16b7c8418da873e6a32aa06b91 2024-11-23T05:42:37,440 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3ae69c16b7c8418da873e6a32aa06b91, entries=150, sequenceid=52, filesize=30.2 K 2024-11-23T05:42:37,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/805e1c6ea2ea499a97b0d8538d5b72ba as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/805e1c6ea2ea499a97b0d8538d5b72ba 2024-11-23T05:42:37,443 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/805e1c6ea2ea499a97b0d8538d5b72ba, entries=150, sequenceid=52, filesize=11.7 K 2024-11-23T05:42:37,444 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/5c10d96ebe78426c875245f83bca34e5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5c10d96ebe78426c875245f83bca34e5 2024-11-23T05:42:37,446 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5c10d96ebe78426c875245f83bca34e5, entries=150, sequenceid=52, filesize=11.7 K 2024-11-23T05:42:37,447 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(3140): Finished flush of dataSize ~60.38 KB/61830, heapSize ~158.91 KB/162720, currentSize=140.89 KB/144270 for f941d6128e412cb98f4108ef00511119 in 1646ms, sequenceid=52, compaction requested=true 2024-11-23T05:42:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:37,447 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=153}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=153 2024-11-23T05:42:37,448 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=153 2024-11-23T05:42:37,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=153, resume processing ppid=152 2024-11-23T05:42:37,450 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=153, ppid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 1.7990 sec 2024-11-23T05:42:37,451 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=152, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=152, table=TestAcidGuarantees in 1.8030 sec 2024-11-23T05:42:37,490 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:37,490 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:37,490 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:42:37,496 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236ef39644121d444c9484a1e36c3eb697_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340556345/Put/seqid=0 2024-11-23T05:42:37,500 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742347_1523 (size=17034) 2024-11-23T05:42:37,501 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:37,503 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 22 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340617498, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340617499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,504 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,504 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 31 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340617499, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,504 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,505 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,506 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236ef39644121d444c9484a1e36c3eb697_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236ef39644121d444c9484a1e36c3eb697_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340617501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,506 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,506 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340617501, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,506 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,507 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,508 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/ae11febd10c14172a4160aacbd33886c, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:37,508 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/ae11febd10c14172a4160aacbd33886c is 175, key is test_row_0/A:col10/1732340556345/Put/seqid=0 2024-11-23T05:42:37,517 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742348_1524 (size=48139) 2024-11-23T05:42:37,608 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,608 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 23 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340617606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,609 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340617606, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,609 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,618 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,618 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340617616, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,618 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 32 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340617618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,620 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340617618, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,620 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,796 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=152 2024-11-23T05:42:37,796 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:37,797 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:37,797 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees 2024-11-23T05:42:37,798 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T05:42:37,798 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:37,799 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=154, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:37,799 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=155, ppid=154, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:37,818 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,818 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 24 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340617815, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,818 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,818 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340617816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,828 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,828 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340617826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,828 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,830 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 33 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340617826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,830 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,831 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 27 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340617826, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:37,831 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:37,906 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T05:42:37,917 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=78, memsize=51.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/ae11febd10c14172a4160aacbd33886c 2024-11-23T05:42:37,923 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d54528007c3649e1b59d9961fb33f614 is 50, key is test_row_0/B:col10/1732340556345/Put/seqid=0 2024-11-23T05:42:37,927 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742349_1525 (size=12001) 2024-11-23T05:42:37,950 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-23T05:42:37,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:37,950 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:37,951 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:37,951 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:37,951 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,103 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-23T05:42:38,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:38,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,103 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,103 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,104 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,115 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T05:42:38,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 25 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340618126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,128 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,128 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340618126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,128 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,128 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,129 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,137 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340618136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,138 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,138 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340618136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,138 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,139 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,139 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 28 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340618136, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,139 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,255 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-23T05:42:38,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,255 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:38,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,256 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,256 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,328 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d54528007c3649e1b59d9961fb33f614 2024-11-23T05:42:38,334 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/9d6244c886004d0c97dc57311a690ff4 is 50, key is test_row_0/C:col10/1732340556345/Put/seqid=0 2024-11-23T05:42:38,337 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742350_1526 (size=12001) 2024-11-23T05:42:38,408 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-23T05:42:38,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:38,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,408 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,408 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,409 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T05:42:38,560 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-23T05:42:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:38,560 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,560 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,561 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,561 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,638 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 26 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340618636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,638 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,639 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340618636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,639 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340618646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340618646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,649 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:38,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 29 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340618646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:38,649 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:38,713 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-23T05:42:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,713 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] handler.RSProcedureHandler(58): pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,713 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=155 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,714 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=155 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:38,758 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=51.44 KB at sequenceid=78 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/9d6244c886004d0c97dc57311a690ff4 2024-11-23T05:42:38,761 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/ae11febd10c14172a4160aacbd33886c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/ae11febd10c14172a4160aacbd33886c 2024-11-23T05:42:38,763 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/ae11febd10c14172a4160aacbd33886c, entries=250, sequenceid=78, filesize=47.0 K 2024-11-23T05:42:38,764 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d54528007c3649e1b59d9961fb33f614 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d54528007c3649e1b59d9961fb33f614 2024-11-23T05:42:38,766 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d54528007c3649e1b59d9961fb33f614, entries=150, sequenceid=78, filesize=11.7 K 2024-11-23T05:42:38,767 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/9d6244c886004d0c97dc57311a690ff4 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/9d6244c886004d0c97dc57311a690ff4 2024-11-23T05:42:38,769 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/9d6244c886004d0c97dc57311a690ff4, entries=150, sequenceid=78, filesize=11.7 K 2024-11-23T05:42:38,770 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~154.31 KB/158010, heapSize ~405 KB/414720, currentSize=53.67 KB/54960 for f941d6128e412cb98f4108ef00511119 in 1280ms, sequenceid=78, compaction requested=true 2024-11-23T05:42:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:38,770 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:38,770 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:38,770 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:38,771 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 158188 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:38,771 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:38,771 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/B is initiating minor compaction (all files) 2024-11-23T05:42:38,771 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/A is initiating minor compaction (all files) 2024-11-23T05:42:38,771 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/B in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,771 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/A in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,771 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d1ad5e8235b4a94a5b5fdd0d79776b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3a91b1c932da40f9ae524240ac8f4835, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3ae69c16b7c8418da873e6a32aa06b91, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/ae11febd10c14172a4160aacbd33886c] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=154.5 K 2024-11-23T05:42:38,771 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/15aa5550c1164693802b7959df6794f3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/6bb3ae26edc948f7bcd4aded0bb7a0ab, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/805e1c6ea2ea499a97b0d8538d5b72ba, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d54528007c3649e1b59d9961fb33f614] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=46.9 K 2024-11-23T05:42:38,771 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,771 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d1ad5e8235b4a94a5b5fdd0d79776b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3a91b1c932da40f9ae524240ac8f4835, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3ae69c16b7c8418da873e6a32aa06b91, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/ae11febd10c14172a4160aacbd33886c] 2024-11-23T05:42:38,771 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 15aa5550c1164693802b7959df6794f3, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732340553501 2024-11-23T05:42:38,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0d1ad5e8235b4a94a5b5fdd0d79776b7, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732340553497 2024-11-23T05:42:38,772 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6bb3ae26edc948f7bcd4aded0bb7a0ab, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732340553527 2024-11-23T05:42:38,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3a91b1c932da40f9ae524240ac8f4835, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732340553527 2024-11-23T05:42:38,772 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 805e1c6ea2ea499a97b0d8538d5b72ba, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340554168 2024-11-23T05:42:38,772 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d54528007c3649e1b59d9961fb33f614, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340556344 2024-11-23T05:42:38,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 3ae69c16b7c8418da873e6a32aa06b91, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340554168 2024-11-23T05:42:38,772 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting ae11febd10c14172a4160aacbd33886c, keycount=250, bloomtype=ROW, size=47.0 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340556330 2024-11-23T05:42:38,778 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#B#compaction#446 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:38,779 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:38,779 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/264c80e833cb46a8942b5026c3db4e1b is 50, key is test_row_0/B:col10/1732340556345/Put/seqid=0 2024-11-23T05:42:38,781 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123f4ece12271d642e997582435607d9b7f_f941d6128e412cb98f4108ef00511119 store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:38,783 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123f4ece12271d642e997582435607d9b7f_f941d6128e412cb98f4108ef00511119, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:38,783 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123f4ece12271d642e997582435607d9b7f_f941d6128e412cb98f4108ef00511119 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:38,796 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742351_1527 (size=12139) 2024-11-23T05:42:38,808 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742352_1528 (size=4469) 2024-11-23T05:42:38,865 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=155 2024-11-23T05:42:38,865 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:38,865 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:42:38,871 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236aa190dc2a184c74a32e873040303b29_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340557499/Put/seqid=0 2024-11-23T05:42:38,873 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742353_1529 (size=12154) 2024-11-23T05:42:38,874 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:38,877 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236aa190dc2a184c74a32e873040303b29_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236aa190dc2a184c74a32e873040303b29_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:38,877 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/914ac9d5864a47f5be2a326a1c833e3e, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:38,878 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/914ac9d5864a47f5be2a326a1c833e3e is 175, key is test_row_0/A:col10/1732340557499/Put/seqid=0 2024-11-23T05:42:38,881 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742354_1530 (size=30955) 2024-11-23T05:42:38,935 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T05:42:39,200 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/264c80e833cb46a8942b5026c3db4e1b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/264c80e833cb46a8942b5026c3db4e1b 2024-11-23T05:42:39,203 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/B of f941d6128e412cb98f4108ef00511119 into 264c80e833cb46a8942b5026c3db4e1b(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:39,203 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:39,203 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/B, priority=12, startTime=1732340558770; duration=0sec 2024-11-23T05:42:39,203 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:39,203 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:B 2024-11-23T05:42:39,203 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:39,204 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48004 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:39,204 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/C is initiating minor compaction (all files) 2024-11-23T05:42:39,204 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/C in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:39,204 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cc46b4b573b04c8d9a907d5bc06e25e1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/aef7ef4925484a0787ab6f1d4b9e5bf5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5c10d96ebe78426c875245f83bca34e5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/9d6244c886004d0c97dc57311a690ff4] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=46.9 K 2024-11-23T05:42:39,204 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting cc46b4b573b04c8d9a907d5bc06e25e1, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=16, earliestPutTs=1732340553501 2024-11-23T05:42:39,205 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting aef7ef4925484a0787ab6f1d4b9e5bf5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=40, earliestPutTs=1732340553527 2024-11-23T05:42:39,205 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5c10d96ebe78426c875245f83bca34e5, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=52, earliestPutTs=1732340554168 2024-11-23T05:42:39,205 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 9d6244c886004d0c97dc57311a690ff4, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340556344 2024-11-23T05:42:39,210 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#A#compaction#447 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:39,210 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/254035676be749728a357d1d70d43294 is 175, key is test_row_0/A:col10/1732340556345/Put/seqid=0 2024-11-23T05:42:39,211 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#C#compaction#449 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:39,211 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/722a6f005d9a437da209f6c5802b2bf1 is 50, key is test_row_0/C:col10/1732340556345/Put/seqid=0 2024-11-23T05:42:39,214 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742356_1532 (size=12139) 2024-11-23T05:42:39,215 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742355_1531 (size=31093) 2024-11-23T05:42:39,281 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=89, memsize=17.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/914ac9d5864a47f5be2a326a1c833e3e 2024-11-23T05:42:39,287 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/66ec397f737941728639d28676188a66 is 50, key is test_row_0/B:col10/1732340557499/Put/seqid=0 2024-11-23T05:42:39,297 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742357_1533 (size=12001) 2024-11-23T05:42:39,618 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/722a6f005d9a437da209f6c5802b2bf1 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/722a6f005d9a437da209f6c5802b2bf1 2024-11-23T05:42:39,620 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/254035676be749728a357d1d70d43294 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/254035676be749728a357d1d70d43294 2024-11-23T05:42:39,622 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/C of f941d6128e412cb98f4108ef00511119 into 722a6f005d9a437da209f6c5802b2bf1(size=11.9 K), total size for store is 11.9 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:39,622 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:39,622 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/C, priority=12, startTime=1732340558770; duration=0sec 2024-11-23T05:42:39,622 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:39,622 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:C 2024-11-23T05:42:39,625 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/A of f941d6128e412cb98f4108ef00511119 into 254035676be749728a357d1d70d43294(size=30.4 K), total size for store is 30.4 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:39,625 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:39,625 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/A, priority=12, startTime=1732340558770; duration=0sec 2024-11-23T05:42:39,625 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:39,625 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:A 2024-11-23T05:42:39,647 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:39,648 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 34 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340619672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340619672, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,676 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,676 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340619673, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,676 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,677 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340619674, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,677 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340619676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,679 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,698 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/66ec397f737941728639d28676188a66 2024-11-23T05:42:39,703 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/bd819b3fc42747d0917ef1cd74b73708 is 50, key is test_row_0/C:col10/1732340557499/Put/seqid=0 2024-11-23T05:42:39,705 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742358_1534 (size=12001) 2024-11-23T05:42:39,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,788 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 35 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340619786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,788 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,788 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340619786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340619786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340619786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,789 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340619787, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,789 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,790 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,946 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T05:42:39,998 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,998 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 36 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340619996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 37 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340619996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 38 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340619996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340619996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,999 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:39,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340619997, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:39,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,000 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,106 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=17.89 KB at sequenceid=89 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/bd819b3fc42747d0917ef1cd74b73708 2024-11-23T05:42:40,109 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/914ac9d5864a47f5be2a326a1c833e3e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/914ac9d5864a47f5be2a326a1c833e3e 2024-11-23T05:42:40,112 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/914ac9d5864a47f5be2a326a1c833e3e, entries=150, sequenceid=89, filesize=30.2 K 2024-11-23T05:42:40,112 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/66ec397f737941728639d28676188a66 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/66ec397f737941728639d28676188a66 2024-11-23T05:42:40,115 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/66ec397f737941728639d28676188a66, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T05:42:40,116 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/bd819b3fc42747d0917ef1cd74b73708 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/bd819b3fc42747d0917ef1cd74b73708 2024-11-23T05:42:40,118 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/bd819b3fc42747d0917ef1cd74b73708, entries=150, sequenceid=89, filesize=11.7 K 2024-11-23T05:42:40,119 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(3140): Finished flush of dataSize ~53.67 KB/54960, heapSize ~141.33 KB/144720, currentSize=147.60 KB/151140 for f941d6128e412cb98f4108ef00511119 in 1253ms, sequenceid=89, compaction requested=false 2024-11-23T05:42:40,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:40,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:40,119 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=155}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=155 2024-11-23T05:42:40,119 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=155 2024-11-23T05:42:40,121 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=155, resume processing ppid=154 2024-11-23T05:42:40,122 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=155, ppid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.3210 sec 2024-11-23T05:42:40,123 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=154, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=154, table=TestAcidGuarantees in 2.3250 sec 2024-11-23T05:42:40,309 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:40,309 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:40,309 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=154.31 KB heapSize=405.05 KB 2024-11-23T05:42:40,314 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340620311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,315 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,315 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340620311, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,315 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,315 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,316 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123185f72455fb9448ab8d9bd1d4eb0efed_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340560309/Put/seqid=0 2024-11-23T05:42:40,316 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,316 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 39 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340620314, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,316 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,319 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742359_1535 (size=12154) 2024-11-23T05:42:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340620315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340620315, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,320 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:40,322 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123185f72455fb9448ab8d9bd1d4eb0efed_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123185f72455fb9448ab8d9bd1d4eb0efed_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:40,323 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0ea8c96e9f09450bae9300c70c8b318f, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:40,323 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0ea8c96e9f09450bae9300c70c8b318f is 175, key is test_row_0/A:col10/1732340560309/Put/seqid=0 2024-11-23T05:42:40,338 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742360_1536 (size=30955) 2024-11-23T05:42:40,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340620426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340620426, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,428 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,428 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 40 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340620427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,429 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,429 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340620427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,429 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340620427, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,429 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,430 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,638 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,638 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340620636, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,638 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 41 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340620637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,639 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,639 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,639 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340620637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,640 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340620637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,640 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,640 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340620637, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,640 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,739 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=120, memsize=55.9 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0ea8c96e9f09450bae9300c70c8b318f 2024-11-23T05:42:40,745 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8e95f8daff5940c1aa2f0495133beca7 is 50, key is test_row_0/B:col10/1732340560309/Put/seqid=0 2024-11-23T05:42:40,749 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742361_1537 (size=12001) 2024-11-23T05:42:40,751 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8e95f8daff5940c1aa2f0495133beca7 2024-11-23T05:42:40,757 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/3911e1ee4f574be6abe74972d6e9b81a is 50, key is test_row_0/C:col10/1732340560309/Put/seqid=0 2024-11-23T05:42:40,760 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742362_1538 (size=12001) 2024-11-23T05:42:40,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 45 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340620946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 42 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340620946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,949 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 46 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340620946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,949 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,949 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,949 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 43 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340620946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,950 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,950 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 44 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340620946, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:40,950 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,160 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=55.91 KB at sequenceid=120 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/3911e1ee4f574be6abe74972d6e9b81a 2024-11-23T05:42:41,164 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0ea8c96e9f09450bae9300c70c8b318f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0ea8c96e9f09450bae9300c70c8b318f 2024-11-23T05:42:41,166 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0ea8c96e9f09450bae9300c70c8b318f, entries=150, sequenceid=120, filesize=30.2 K 2024-11-23T05:42:41,167 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8e95f8daff5940c1aa2f0495133beca7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8e95f8daff5940c1aa2f0495133beca7 2024-11-23T05:42:41,169 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8e95f8daff5940c1aa2f0495133beca7, entries=150, sequenceid=120, filesize=11.7 K 2024-11-23T05:42:41,170 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/3911e1ee4f574be6abe74972d6e9b81a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/3911e1ee4f574be6abe74972d6e9b81a 2024-11-23T05:42:41,172 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/3911e1ee4f574be6abe74972d6e9b81a, entries=150, sequenceid=120, filesize=11.7 K 2024-11-23T05:42:41,173 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~167.72 KB/171750, heapSize ~440.16 KB/450720, currentSize=33.54 KB/34350 for f941d6128e412cb98f4108ef00511119 in 864ms, sequenceid=120, compaction requested=true 2024-11-23T05:42:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:41,173 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:41,173 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:41,173 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:41,174 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93003 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:41,174 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/A is initiating minor compaction (all files) 2024-11-23T05:42:41,174 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/A in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:41,174 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/254035676be749728a357d1d70d43294, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/914ac9d5864a47f5be2a326a1c833e3e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0ea8c96e9f09450bae9300c70c8b318f] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=90.8 K 2024-11-23T05:42:41,174 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:41,174 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/254035676be749728a357d1d70d43294, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/914ac9d5864a47f5be2a326a1c833e3e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0ea8c96e9f09450bae9300c70c8b318f] 2024-11-23T05:42:41,174 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:41,174 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/B is initiating minor compaction (all files) 2024-11-23T05:42:41,174 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/B in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:41,174 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/264c80e833cb46a8942b5026c3db4e1b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/66ec397f737941728639d28676188a66, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8e95f8daff5940c1aa2f0495133beca7] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=35.3 K 2024-11-23T05:42:41,175 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 254035676be749728a357d1d70d43294, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340556344 2024-11-23T05:42:41,175 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 264c80e833cb46a8942b5026c3db4e1b, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340556344 2024-11-23T05:42:41,175 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 914ac9d5864a47f5be2a326a1c833e3e, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732340557498 2024-11-23T05:42:41,175 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 66ec397f737941728639d28676188a66, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732340557498 2024-11-23T05:42:41,175 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 0ea8c96e9f09450bae9300c70c8b318f, keycount=150, bloomtype=ROW, size=30.2 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732340559670 2024-11-23T05:42:41,175 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8e95f8daff5940c1aa2f0495133beca7, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732340559670 2024-11-23T05:42:41,179 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:41,181 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#B#compaction#455 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:41,181 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123864d155772794797a88dcf94b4f04ece_f941d6128e412cb98f4108ef00511119 store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:41,181 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/ee87403090484184a5018345a997696e is 50, key is test_row_0/B:col10/1732340560309/Put/seqid=0 2024-11-23T05:42:41,183 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123864d155772794797a88dcf94b4f04ece_f941d6128e412cb98f4108ef00511119, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:41,183 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123864d155772794797a88dcf94b4f04ece_f941d6128e412cb98f4108ef00511119 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:41,185 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742363_1539 (size=12241) 2024-11-23T05:42:41,189 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742364_1540 (size=4469) 2024-11-23T05:42:41,194 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/ee87403090484184a5018345a997696e as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/ee87403090484184a5018345a997696e 2024-11-23T05:42:41,197 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f941d6128e412cb98f4108ef00511119/B of f941d6128e412cb98f4108ef00511119 into ee87403090484184a5018345a997696e(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:41,198 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:41,198 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/B, priority=13, startTime=1732340561173; duration=0sec 2024-11-23T05:42:41,198 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:41,198 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:B 2024-11-23T05:42:41,198 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:41,198 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36141 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:41,198 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/C is initiating minor compaction (all files) 2024-11-23T05:42:41,198 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/C in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:41,198 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/722a6f005d9a437da209f6c5802b2bf1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/bd819b3fc42747d0917ef1cd74b73708, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/3911e1ee4f574be6abe74972d6e9b81a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=35.3 K 2024-11-23T05:42:41,199 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 722a6f005d9a437da209f6c5802b2bf1, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=78, earliestPutTs=1732340556344 2024-11-23T05:42:41,200 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting bd819b3fc42747d0917ef1cd74b73708, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=89, earliestPutTs=1732340557498 2024-11-23T05:42:41,200 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3911e1ee4f574be6abe74972d6e9b81a, keycount=150, bloomtype=ROW, size=11.7 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732340559670 2024-11-23T05:42:41,204 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#C#compaction#457 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:41,205 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/cf667d8128a247419eca982eb6596b9f is 50, key is test_row_0/C:col10/1732340560309/Put/seqid=0 2024-11-23T05:42:41,208 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742365_1541 (size=12241) 2024-11-23T05:42:41,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:41,459 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:41,459 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=53.67 KB heapSize=141.38 KB 2024-11-23T05:42:41,465 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237f66f89c6e67423e926ec905ce4b9942_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340561458/Put/seqid=0 2024-11-23T05:42:41,468 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742366_1542 (size=17284) 2024-11-23T05:42:41,539 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,539 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340621509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,539 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 48 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340621509, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,539 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340621539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340621539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,543 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,543 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340621539, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,543 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,544 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,590 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#A#compaction#456 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:41,590 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/a2f4e920490b45e2999e147bd1f1c155 is 175, key is test_row_0/A:col10/1732340560309/Put/seqid=0 2024-11-23T05:42:41,593 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742367_1543 (size=31195) 2024-11-23T05:42:41,611 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/cf667d8128a247419eca982eb6596b9f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cf667d8128a247419eca982eb6596b9f 2024-11-23T05:42:41,614 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f941d6128e412cb98f4108ef00511119/C of f941d6128e412cb98f4108ef00511119 into cf667d8128a247419eca982eb6596b9f(size=12.0 K), total size for store is 12.0 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:41,614 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:41,614 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/C, priority=13, startTime=1732340561173; duration=0sec 2024-11-23T05:42:41,614 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:41,614 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:C 2024-11-23T05:42:41,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 49 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340621646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340621646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,648 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,648 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340621646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,648 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340621646, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,649 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,649 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340621647, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,649 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,858 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 50 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340621856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,858 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340621856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340621856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340621856, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,859 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340621857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,859 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:41,860 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:41,869 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:41,871 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411237f66f89c6e67423e926ec905ce4b9942_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237f66f89c6e67423e926ec905ce4b9942_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:41,872 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/95634794404f45d4aa7276e132bb62c6, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:41,872 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/95634794404f45d4aa7276e132bb62c6 is 175, key is test_row_0/A:col10/1732340561458/Put/seqid=0 2024-11-23T05:42:41,875 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742368_1544 (size=48389) 2024-11-23T05:42:41,966 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=154 2024-11-23T05:42:41,966 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:41,966 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees 2024-11-23T05:42:41,967 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T05:42:41,968 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:41,968 INFO [PEWorker-4 {}] procedure.FlushTableProcedure(91): pid=156, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:41,968 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=157, ppid=156, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:42,002 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/a2f4e920490b45e2999e147bd1f1c155 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a2f4e920490b45e2999e147bd1f1c155 2024-11-23T05:42:42,006 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f941d6128e412cb98f4108ef00511119/A of f941d6128e412cb98f4108ef00511119 into a2f4e920490b45e2999e147bd1f1c155(size=30.5 K), total size for store is 30.5 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:42,006 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:42,006 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/A, priority=13, startTime=1732340561173; duration=0sec 2024-11-23T05:42:42,006 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:42,006 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:A 2024-11-23T05:42:42,075 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T05:42:42,120 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T05:42:42,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:42,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,120 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,120 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,121 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 51 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340622166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340622166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340622166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,168 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340622166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,169 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,169 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340622166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,169 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,273 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T05:42:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,273 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,273 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,273 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,275 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=134, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/95634794404f45d4aa7276e132bb62c6 2024-11-23T05:42:42,280 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d89b7efb7ead441a8632728d097fece3 is 50, key is test_row_0/B:col10/1732340561458/Put/seqid=0 2024-11-23T05:42:42,283 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742369_1545 (size=12151) 2024-11-23T05:42:42,283 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d89b7efb7ead441a8632728d097fece3 2024-11-23T05:42:42,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T05:42:42,288 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/1419234c028e48dcb187470ecb914c67 is 50, key is test_row_0/C:col10/1732340561458/Put/seqid=0 2024-11-23T05:42:42,290 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742370_1546 (size=12151) 2024-11-23T05:42:42,425 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T05:42:42,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:42,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,425 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,425 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,577 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T05:42:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,578 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] handler.RSProcedureHandler(58): pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=157 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,578 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=157 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:42,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T05:42:42,677 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,677 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 52 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340622675, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340622676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,678 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 53 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340622676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,678 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,678 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340622676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,679 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 54 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340622676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:42,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:42,691 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=134 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/1419234c028e48dcb187470ecb914c67 2024-11-23T05:42:42,693 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/95634794404f45d4aa7276e132bb62c6 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/95634794404f45d4aa7276e132bb62c6 2024-11-23T05:42:42,696 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/95634794404f45d4aa7276e132bb62c6, entries=250, sequenceid=134, filesize=47.3 K 2024-11-23T05:42:42,697 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d89b7efb7ead441a8632728d097fece3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d89b7efb7ead441a8632728d097fece3 2024-11-23T05:42:42,699 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d89b7efb7ead441a8632728d097fece3, entries=150, sequenceid=134, filesize=11.9 K 2024-11-23T05:42:42,699 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/1419234c028e48dcb187470ecb914c67 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/1419234c028e48dcb187470ecb914c67 2024-11-23T05:42:42,702 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/1419234c028e48dcb187470ecb914c67, entries=150, sequenceid=134, filesize=11.9 K 2024-11-23T05:42:42,702 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=140.89 KB/144270 for f941d6128e412cb98f4108ef00511119 in 1243ms, sequenceid=134, compaction requested=false 2024-11-23T05:42:42,702 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:42,730 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=157 2024-11-23T05:42:42,730 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:42,730 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=140.89 KB heapSize=369.89 KB 2024-11-23T05:42:42,736 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112399ddf95f65c342a2afd2acc9618eb866_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340561508/Put/seqid=0 2024-11-23T05:42:42,746 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742371_1547 (size=12304) 2024-11-23T05:42:43,106 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T05:42:43,147 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:43,149 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112399ddf95f65c342a2afd2acc9618eb866_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112399ddf95f65c342a2afd2acc9618eb866_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:43,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/06d9004ac38142938185864cc3ff1061, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:43,150 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/06d9004ac38142938185864cc3ff1061 is 175, key is test_row_0/A:col10/1732340561508/Put/seqid=0 2024-11-23T05:42:43,153 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742372_1548 (size=31105) 2024-11-23T05:42:43,554 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=160, memsize=47.0 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/06d9004ac38142938185864cc3ff1061 2024-11-23T05:42:43,559 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/5e8a39ac29944a2aacaadac3377a0f29 is 50, key is test_row_0/B:col10/1732340561508/Put/seqid=0 2024-11-23T05:42:43,562 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742373_1549 (size=12151) 2024-11-23T05:42:43,562 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/5e8a39ac29944a2aacaadac3377a0f29 2024-11-23T05:42:43,567 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/62c7c8050c3a409ab3ad1d82991469fd is 50, key is test_row_0/C:col10/1732340561508/Put/seqid=0 2024-11-23T05:42:43,570 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742374_1550 (size=12151) 2024-11-23T05:42:43,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:43,689 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:43,695 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,695 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340623691, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,696 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,698 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 55 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340623695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340623695, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340623696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,699 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,699 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 59 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340623696, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,699 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,700 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,810 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,810 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340623806, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340623807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 57 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340623807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,811 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,811 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,811 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 60 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340623807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,812 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,812 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 56 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340623807, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:43,812 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:43,970 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=46.96 KB at sequenceid=160 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/62c7c8050c3a409ab3ad1d82991469fd 2024-11-23T05:42:43,973 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/06d9004ac38142938185864cc3ff1061 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/06d9004ac38142938185864cc3ff1061 2024-11-23T05:42:43,976 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/06d9004ac38142938185864cc3ff1061, entries=150, sequenceid=160, filesize=30.4 K 2024-11-23T05:42:43,976 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/5e8a39ac29944a2aacaadac3377a0f29 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/5e8a39ac29944a2aacaadac3377a0f29 2024-11-23T05:42:43,979 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/5e8a39ac29944a2aacaadac3377a0f29, entries=150, sequenceid=160, filesize=11.9 K 2024-11-23T05:42:43,979 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/62c7c8050c3a409ab3ad1d82991469fd as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/62c7c8050c3a409ab3ad1d82991469fd 2024-11-23T05:42:43,982 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/62c7c8050c3a409ab3ad1d82991469fd, entries=150, sequenceid=160, filesize=11.9 K 2024-11-23T05:42:43,983 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(3140): Finished flush of dataSize ~140.89 KB/144270, heapSize ~369.84 KB/378720, currentSize=60.38 KB/61830 for f941d6128e412cb98f4108ef00511119 in 1253ms, sequenceid=160, compaction requested=true 2024-11-23T05:42:43,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:43,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:43,983 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=157}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=157 2024-11-23T05:42:43,983 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=157 2024-11-23T05:42:43,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=157, resume processing ppid=156 2024-11-23T05:42:43,987 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=157, ppid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.0170 sec 2024-11-23T05:42:43,988 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=156, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=156, table=TestAcidGuarantees in 2.0210 sec 2024-11-23T05:42:44,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:44,018 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:44,018 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:42:44,027 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112331fe8caed1ab4c8483d87dfd9f00e349_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340563690/Put/seqid=0 2024-11-23T05:42:44,032 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742375_1551 (size=12304) 2024-11-23T05:42:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,052 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340624047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,052 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340624047, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340624048, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,053 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 61 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340624049, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,053 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,057 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340624052, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,057 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,116 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=156 2024-11-23T05:42:44,116 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:44,116 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:44,117 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees 2024-11-23T05:42:44,118 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T05:42:44,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:44,118 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=158, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:44,119 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=159, ppid=158, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:44,157 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,157 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340624156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340624156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340624156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,158 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 62 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340624156, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,158 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,168 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,168 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340624166, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,168 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,226 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T05:42:44,270 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:44,270 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,271 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,271 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,271 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340624366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,369 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,369 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,369 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340624366, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,370 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 63 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340624367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,370 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,370 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340624367, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,370 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,378 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,379 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340624376, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,379 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,423 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,423 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:44,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,424 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,424 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,424 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,432 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:44,435 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112331fe8caed1ab4c8483d87dfd9f00e349_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112331fe8caed1ab4c8483d87dfd9f00e349_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:44,435 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T05:42:44,435 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/50d05d5baeec4d68b4c06bdb7c06e6ba, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:44,436 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/50d05d5baeec4d68b4c06bdb7c06e6ba is 175, key is test_row_0/A:col10/1732340563690/Put/seqid=0 2024-11-23T05:42:44,438 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742376_1552 (size=31105) 2024-11-23T05:42:44,576 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,576 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,576 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,577 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340624676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340624676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,679 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,679 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 64 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340624676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,679 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,680 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,680 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340624676, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,680 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,689 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:44,689 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340624686, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:44,689 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:44,731 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:44,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:44,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,732 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,732 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,733 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T05:42:44,839 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=173, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/50d05d5baeec4d68b4c06bdb7c06e6ba 2024-11-23T05:42:44,845 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/3b1e0035440a491cb316de3afc1a39cc is 50, key is test_row_0/B:col10/1732340563690/Put/seqid=0 2024-11-23T05:42:44,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742377_1553 (size=12151) 2024-11-23T05:42:44,848 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/3b1e0035440a491cb316de3afc1a39cc 2024-11-23T05:42:44,852 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/6ab69063359d4928aab611f4b76a1780 is 50, key is test_row_0/C:col10/1732340563690/Put/seqid=0 2024-11-23T05:42:44,855 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742378_1554 (size=12151) 2024-11-23T05:42:44,884 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:44,884 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:44,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:44,885 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,885 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:44,885 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:45,037 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:45,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:45,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,037 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:45,037 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:45,038 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:45,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 66 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340625185, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:45,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,187 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340625186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:45,187 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:45,187 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 65 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340625186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:45,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340625186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,188 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:45,189 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:45,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:45,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,190 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] handler.RSProcedureHandler(58): pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:45,190 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=159 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:45,190 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=159 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:45,197 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:45,197 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 67 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340625196, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:45,197 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:45,255 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=173 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/6ab69063359d4928aab611f4b76a1780 2024-11-23T05:42:45,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T05:42:45,258 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/50d05d5baeec4d68b4c06bdb7c06e6ba as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/50d05d5baeec4d68b4c06bdb7c06e6ba 2024-11-23T05:42:45,261 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/50d05d5baeec4d68b4c06bdb7c06e6ba, entries=150, sequenceid=173, filesize=30.4 K 2024-11-23T05:42:45,262 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/3b1e0035440a491cb316de3afc1a39cc as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3b1e0035440a491cb316de3afc1a39cc 2024-11-23T05:42:45,265 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3b1e0035440a491cb316de3afc1a39cc, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T05:42:45,265 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/6ab69063359d4928aab611f4b76a1780 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/6ab69063359d4928aab611f4b76a1780 2024-11-23T05:42:45,267 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/6ab69063359d4928aab611f4b76a1780, entries=150, sequenceid=173, filesize=11.9 K 2024-11-23T05:42:45,268 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=134.18 KB/137400 for f941d6128e412cb98f4108ef00511119 in 1250ms, sequenceid=173, compaction requested=true 2024-11-23T05:42:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:45,268 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:45,268 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:45,268 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:45,269 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 141794 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:45,269 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/A is initiating minor compaction (all files) 2024-11-23T05:42:45,269 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/A in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,269 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a2f4e920490b45e2999e147bd1f1c155, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/95634794404f45d4aa7276e132bb62c6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/06d9004ac38142938185864cc3ff1061, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/50d05d5baeec4d68b4c06bdb7c06e6ba] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=138.5 K 2024-11-23T05:42:45,269 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,269 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a2f4e920490b45e2999e147bd1f1c155, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/95634794404f45d4aa7276e132bb62c6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/06d9004ac38142938185864cc3ff1061, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/50d05d5baeec4d68b4c06bdb7c06e6ba] 2024-11-23T05:42:45,270 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:45,270 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/B is initiating minor compaction (all files) 2024-11-23T05:42:45,270 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/B in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,270 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/ee87403090484184a5018345a997696e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d89b7efb7ead441a8632728d097fece3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/5e8a39ac29944a2aacaadac3377a0f29, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3b1e0035440a491cb316de3afc1a39cc] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=47.6 K 2024-11-23T05:42:45,270 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a2f4e920490b45e2999e147bd1f1c155, keycount=150, bloomtype=ROW, size=30.5 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732340559670 2024-11-23T05:42:45,270 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting ee87403090484184a5018345a997696e, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732340559670 2024-11-23T05:42:45,270 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d89b7efb7ead441a8632728d097fece3, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732340561458 2024-11-23T05:42:45,271 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 95634794404f45d4aa7276e132bb62c6, keycount=250, bloomtype=ROW, size=47.3 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732340561457 2024-11-23T05:42:45,271 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5e8a39ac29944a2aacaadac3377a0f29, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732340561508 2024-11-23T05:42:45,271 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 06d9004ac38142938185864cc3ff1061, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732340561508 2024-11-23T05:42:45,271 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 50d05d5baeec4d68b4c06bdb7c06e6ba, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732340563690 2024-11-23T05:42:45,271 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3b1e0035440a491cb316de3afc1a39cc, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732340563690 2024-11-23T05:42:45,276 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:45,277 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#B#compaction#467 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:45,278 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/b85b63488771425a8418cd978fb063f3 is 50, key is test_row_0/B:col10/1732340563690/Put/seqid=0 2024-11-23T05:42:45,283 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123cd073d60d69e4792a521b725dd114821_f941d6128e412cb98f4108ef00511119 store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:45,286 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123cd073d60d69e4792a521b725dd114821_f941d6128e412cb98f4108ef00511119, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:45,286 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123cd073d60d69e4792a521b725dd114821_f941d6128e412cb98f4108ef00511119 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:45,287 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742379_1555 (size=12527) 2024-11-23T05:42:45,289 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742380_1556 (size=4469) 2024-11-23T05:42:45,292 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/b85b63488771425a8418cd978fb063f3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/b85b63488771425a8418cd978fb063f3 2024-11-23T05:42:45,295 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/B of f941d6128e412cb98f4108ef00511119 into b85b63488771425a8418cd978fb063f3(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:45,295 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:45,295 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/B, priority=12, startTime=1732340565268; duration=0sec 2024-11-23T05:42:45,296 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:45,296 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:B 2024-11-23T05:42:45,296 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:45,296 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 48694 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:45,297 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/C is initiating minor compaction (all files) 2024-11-23T05:42:45,297 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/C in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,297 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cf667d8128a247419eca982eb6596b9f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/1419234c028e48dcb187470ecb914c67, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/62c7c8050c3a409ab3ad1d82991469fd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/6ab69063359d4928aab611f4b76a1780] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=47.6 K 2024-11-23T05:42:45,297 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting cf667d8128a247419eca982eb6596b9f, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=120, earliestPutTs=1732340559670 2024-11-23T05:42:45,297 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 1419234c028e48dcb187470ecb914c67, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=134, earliestPutTs=1732340561458 2024-11-23T05:42:45,297 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 62c7c8050c3a409ab3ad1d82991469fd, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=160, earliestPutTs=1732340561508 2024-11-23T05:42:45,297 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 6ab69063359d4928aab611f4b76a1780, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732340563690 2024-11-23T05:42:45,303 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#C#compaction#469 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:45,304 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/8274dc580a2e48d0998e7e26163ce326 is 50, key is test_row_0/C:col10/1732340563690/Put/seqid=0 2024-11-23T05:42:45,307 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742381_1557 (size=12527) 2024-11-23T05:42:45,342 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=159 2024-11-23T05:42:45,342 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:45,342 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:42:45,347 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b5c6d74e72ee42ad8fb7c242c1a0f9c3_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340564045/Put/seqid=0 2024-11-23T05:42:45,350 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742382_1558 (size=12304) 2024-11-23T05:42:45,690 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#A#compaction#468 average throughput is 0.06 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:45,691 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/9d461d7d5405466b99039fb11480c36d is 175, key is test_row_0/A:col10/1732340563690/Put/seqid=0 2024-11-23T05:42:45,693 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742383_1559 (size=31481) 2024-11-23T05:42:45,710 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/8274dc580a2e48d0998e7e26163ce326 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/8274dc580a2e48d0998e7e26163ce326 2024-11-23T05:42:45,713 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/C of f941d6128e412cb98f4108ef00511119 into 8274dc580a2e48d0998e7e26163ce326(size=12.2 K), total size for store is 12.2 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:45,713 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:45,714 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/C, priority=12, startTime=1732340565268; duration=0sec 2024-11-23T05:42:45,714 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:45,714 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:C 2024-11-23T05:42:45,751 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:45,754 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123b5c6d74e72ee42ad8fb7c242c1a0f9c3_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b5c6d74e72ee42ad8fb7c242c1a0f9c3_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:45,754 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/2bd4547f122242d082d34c42803089ef, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:45,755 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/2bd4547f122242d082d34c42803089ef is 175, key is test_row_0/A:col10/1732340564045/Put/seqid=0 2024-11-23T05:42:45,757 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742384_1560 (size=31105) 2024-11-23T05:42:45,758 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=197, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/2bd4547f122242d082d34c42803089ef 2024-11-23T05:42:45,762 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8dca9b0a0a5a482eaa79426502e8c42c is 50, key is test_row_0/B:col10/1732340564045/Put/seqid=0 2024-11-23T05:42:45,765 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742385_1561 (size=12151) 2024-11-23T05:42:46,097 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/9d461d7d5405466b99039fb11480c36d as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/9d461d7d5405466b99039fb11480c36d 2024-11-23T05:42:46,100 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/A of f941d6128e412cb98f4108ef00511119 into 9d461d7d5405466b99039fb11480c36d(size=30.7 K), total size for store is 30.7 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:46,100 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:46,100 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/A, priority=12, startTime=1732340565268; duration=0sec 2024-11-23T05:42:46,100 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:46,100 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:A 2024-11-23T05:42:46,166 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8dca9b0a0a5a482eaa79426502e8c42c 2024-11-23T05:42:46,172 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/4bd28f2dfd674dc283f5fa196a04a2f2 is 50, key is test_row_0/C:col10/1732340564045/Put/seqid=0 2024-11-23T05:42:46,175 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742386_1562 (size=12151) 2024-11-23T05:42:46,198 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:46,198 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:46,208 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,208 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 68 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340626206, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,209 WARN [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2157 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,209 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,210 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,210 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340626207, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,210 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,211 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,211 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340626208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 69 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340626208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,212 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,212 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340626208, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,212 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,266 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T05:42:46,317 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,317 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 74 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340626316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,317 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340626316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 70 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340626316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,319 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,319 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,319 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340626316, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,320 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 75 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340626526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,527 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,527 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,527 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 72 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340626526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 71 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340626526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,528 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,528 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 73 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340626526, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,528 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,576 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=197 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/4bd28f2dfd674dc283f5fa196a04a2f2 2024-11-23T05:42:46,578 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/2bd4547f122242d082d34c42803089ef as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/2bd4547f122242d082d34c42803089ef 2024-11-23T05:42:46,581 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/2bd4547f122242d082d34c42803089ef, entries=150, sequenceid=197, filesize=30.4 K 2024-11-23T05:42:46,582 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8dca9b0a0a5a482eaa79426502e8c42c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8dca9b0a0a5a482eaa79426502e8c42c 2024-11-23T05:42:46,585 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8dca9b0a0a5a482eaa79426502e8c42c, entries=150, sequenceid=197, filesize=11.9 K 2024-11-23T05:42:46,585 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/4bd28f2dfd674dc283f5fa196a04a2f2 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/4bd28f2dfd674dc283f5fa196a04a2f2 2024-11-23T05:42:46,588 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/4bd28f2dfd674dc283f5fa196a04a2f2, entries=150, sequenceid=197, filesize=11.9 K 2024-11-23T05:42:46,588 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f941d6128e412cb98f4108ef00511119 in 1246ms, sequenceid=197, compaction requested=false 2024-11-23T05:42:46,588 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:46,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:46,589 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=159}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=159 2024-11-23T05:42:46,589 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=159 2024-11-23T05:42:46,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=159, resume processing ppid=158 2024-11-23T05:42:46,591 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=159, ppid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.4710 sec 2024-11-23T05:42:46,592 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=158, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=158, table=TestAcidGuarantees in 2.4750 sec 2024-11-23T05:42:46,838 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:46,838 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:46,838 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:42:46,843 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239cb041d77fb6464abd3c167184c4d97f_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340566838/Put/seqid=0 2024-11-23T05:42:46,847 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742387_1563 (size=12304) 2024-11-23T05:42:46,847 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:46,850 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411239cb041d77fb6464abd3c167184c4d97f_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239cb041d77fb6464abd3c167184c4d97f_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:46,850 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/a0734cd7fdda4199bc61d4dc856af572, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:46,851 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/a0734cd7fdda4199bc61d4dc856af572 is 175, key is test_row_0/A:col10/1732340566838/Put/seqid=0 2024-11-23T05:42:46,853 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742388_1564 (size=31105) 2024-11-23T05:42:46,863 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,863 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340626857, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,863 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 77 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340626861, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340626862, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,867 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,867 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340626863, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,867 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,968 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,968 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340626965, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,968 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 78 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340626976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340626976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,978 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,978 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:46,978 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340626976, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:46,979 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,177 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,177 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340627175, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,177 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,188 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 79 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340627186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,188 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340627186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,189 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,189 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340627186, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,189 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,254 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=214, memsize=26.8 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/a0734cd7fdda4199bc61d4dc856af572 2024-11-23T05:42:47,258 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/3aba7490273b4a1fb774881fd56326ad is 50, key is test_row_0/B:col10/1732340566838/Put/seqid=0 2024-11-23T05:42:47,261 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742389_1565 (size=12151) 2024-11-23T05:42:47,488 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,488 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340627486, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,489 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,497 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,497 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 80 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340627496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340627496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,498 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,498 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340627496, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,498 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:47,662 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/3aba7490273b4a1fb774881fd56326ad 2024-11-23T05:42:47,666 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/5d503136f51f435990464e1c8f60272a is 50, key is test_row_0/C:col10/1732340566838/Put/seqid=0 2024-11-23T05:42:47,669 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742390_1566 (size=12151) 2024-11-23T05:42:47,999 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:47,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340627996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:47,999 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,008 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340628006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340628006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,009 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:48,009 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340628006, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,009 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,069 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=26.84 KB at sequenceid=214 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/5d503136f51f435990464e1c8f60272a 2024-11-23T05:42:48,073 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/a0734cd7fdda4199bc61d4dc856af572 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a0734cd7fdda4199bc61d4dc856af572 2024-11-23T05:42:48,075 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a0734cd7fdda4199bc61d4dc856af572, entries=150, sequenceid=214, filesize=30.4 K 2024-11-23T05:42:48,076 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/3aba7490273b4a1fb774881fd56326ad as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3aba7490273b4a1fb774881fd56326ad 2024-11-23T05:42:48,078 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3aba7490273b4a1fb774881fd56326ad, entries=150, sequenceid=214, filesize=11.9 K 2024-11-23T05:42:48,079 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/5d503136f51f435990464e1c8f60272a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5d503136f51f435990464e1c8f60272a 2024-11-23T05:42:48,082 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5d503136f51f435990464e1c8f60272a, entries=150, sequenceid=214, filesize=11.9 K 2024-11-23T05:42:48,082 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~80.51 KB/82440, heapSize ~211.64 KB/216720, currentSize=120.76 KB/123660 for f941d6128e412cb98f4108ef00511119 in 1244ms, sequenceid=214, compaction requested=true 2024-11-23T05:42:48,082 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:48,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:48,082 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:48,083 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:48,083 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:48,083 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:48,083 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 93691 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:48,083 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:48,083 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/B is initiating minor compaction (all files) 2024-11-23T05:42:48,083 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/A is initiating minor compaction (all files) 2024-11-23T05:42:48,083 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/A in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,084 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/9d461d7d5405466b99039fb11480c36d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/2bd4547f122242d082d34c42803089ef, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a0734cd7fdda4199bc61d4dc856af572] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=91.5 K 2024-11-23T05:42:48,084 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=13 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,084 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/9d461d7d5405466b99039fb11480c36d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/2bd4547f122242d082d34c42803089ef, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a0734cd7fdda4199bc61d4dc856af572] 2024-11-23T05:42:48,084 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 9d461d7d5405466b99039fb11480c36d, keycount=150, bloomtype=ROW, size=30.7 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732340563690 2024-11-23T05:42:48,084 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/B in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,084 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/b85b63488771425a8418cd978fb063f3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8dca9b0a0a5a482eaa79426502e8c42c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3aba7490273b4a1fb774881fd56326ad] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=36.0 K 2024-11-23T05:42:48,084 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 2bd4547f122242d082d34c42803089ef, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732340564045 2024-11-23T05:42:48,084 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b85b63488771425a8418cd978fb063f3, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732340563690 2024-11-23T05:42:48,084 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting a0734cd7fdda4199bc61d4dc856af572, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732340566206 2024-11-23T05:42:48,084 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8dca9b0a0a5a482eaa79426502e8c42c, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732340564045 2024-11-23T05:42:48,085 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 3aba7490273b4a1fb774881fd56326ad, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732340566206 2024-11-23T05:42:48,090 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:48,092 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e2024112321fea4b9192e4064affc1a264d55919d_f941d6128e412cb98f4108ef00511119 store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:48,092 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#B#compaction#477 average throughput is 3.28 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:48,092 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/97ce7dbb38834f98bc3123df0f9dd013 is 50, key is test_row_0/B:col10/1732340566838/Put/seqid=0 2024-11-23T05:42:48,093 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e2024112321fea4b9192e4064affc1a264d55919d_f941d6128e412cb98f4108ef00511119, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:48,093 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112321fea4b9192e4064affc1a264d55919d_f941d6128e412cb98f4108ef00511119 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:48,095 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742391_1567 (size=12629) 2024-11-23T05:42:48,096 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742392_1568 (size=4469) 2024-11-23T05:42:48,097 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#A#compaction#476 average throughput is 3.49 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:48,098 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/d4e0af6a317841d8bbae2c4b68db6836 is 175, key is test_row_0/A:col10/1732340566838/Put/seqid=0 2024-11-23T05:42:48,101 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742393_1569 (size=31583) 2024-11-23T05:42:48,101 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/97ce7dbb38834f98bc3123df0f9dd013 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/97ce7dbb38834f98bc3123df0f9dd013 2024-11-23T05:42:48,105 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/d4e0af6a317841d8bbae2c4b68db6836 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/d4e0af6a317841d8bbae2c4b68db6836 2024-11-23T05:42:48,105 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f941d6128e412cb98f4108ef00511119/B of f941d6128e412cb98f4108ef00511119 into 97ce7dbb38834f98bc3123df0f9dd013(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:48,105 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:48,105 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/B, priority=13, startTime=1732340568083; duration=0sec 2024-11-23T05:42:48,105 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:48,105 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:B 2024-11-23T05:42:48,105 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 3 store files, 0 compacting, 3 eligible, 16 blocking 2024-11-23T05:42:48,107 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 3 files of size 36829 starting at candidate #0 after considering 1 permutations with 1 in ratio 2024-11-23T05:42:48,107 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/C is initiating minor compaction (all files) 2024-11-23T05:42:48,107 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/C in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,107 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/8274dc580a2e48d0998e7e26163ce326, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/4bd28f2dfd674dc283f5fa196a04a2f2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5d503136f51f435990464e1c8f60272a] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=36.0 K 2024-11-23T05:42:48,107 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 8274dc580a2e48d0998e7e26163ce326, keycount=150, bloomtype=ROW, size=12.2 K, encoding=NONE, compression=NONE, seqNum=173, earliestPutTs=1732340563690 2024-11-23T05:42:48,107 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 4bd28f2dfd674dc283f5fa196a04a2f2, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=197, earliestPutTs=1732340564045 2024-11-23T05:42:48,108 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 5d503136f51f435990464e1c8f60272a, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732340566206 2024-11-23T05:42:48,108 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f941d6128e412cb98f4108ef00511119/A of f941d6128e412cb98f4108ef00511119 into d4e0af6a317841d8bbae2c4b68db6836(size=30.8 K), total size for store is 30.8 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:48,108 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:48,108 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/A, priority=13, startTime=1732340568082; duration=0sec 2024-11-23T05:42:48,108 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:48,108 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:A 2024-11-23T05:42:48,112 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#C#compaction#478 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:48,113 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/642b339d2548496e8a436d83de2c5a05 is 50, key is test_row_0/C:col10/1732340566838/Put/seqid=0 2024-11-23T05:42:48,115 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742394_1570 (size=12629) 2024-11-23T05:42:48,118 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/642b339d2548496e8a436d83de2c5a05 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/642b339d2548496e8a436d83de2c5a05 2024-11-23T05:42:48,122 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 3 (all) file(s) in f941d6128e412cb98f4108ef00511119/C of f941d6128e412cb98f4108ef00511119 into 642b339d2548496e8a436d83de2c5a05(size=12.3 K), total size for store is 12.3 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:48,122 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:48,122 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/C, priority=13, startTime=1732340568083; duration=0sec 2024-11-23T05:42:48,122 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:48,122 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:C 2024-11-23T05:42:48,238 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:48,238 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:48,239 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=127.47 KB heapSize=334.73 KB 2024-11-23T05:42:48,245 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236e31977e02894ddaa60bb1c3ec11c457_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340566861/Put/seqid=0 2024-11-23T05:42:48,248 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742395_1571 (size=14794) 2024-11-23T05:42:48,275 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:48,275 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 81 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340628272, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,275 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,286 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=158 2024-11-23T05:42:48,286 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:48,286 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:48,287 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees 2024-11-23T05:42:48,288 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T05:42:48,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:48,288 INFO [PEWorker-1 {}] procedure.FlushTableProcedure(91): pid=160, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:48,288 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=161, ppid=160, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:48,388 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:48,388 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340628385, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,388 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,396 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T05:42:48,440 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:48,440 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,440 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,441 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,441 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,593 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,593 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,593 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,594 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,598 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:48,599 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340628596, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,599 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,605 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T05:42:48,648 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:48,651 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e202411236e31977e02894ddaa60bb1c3ec11c457_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236e31977e02894ddaa60bb1c3ec11c457_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:48,652 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/448695501d6842c2ad142a0903d324db, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:48,652 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/448695501d6842c2ad142a0903d324db is 175, key is test_row_0/A:col10/1732340566861/Put/seqid=0 2024-11-23T05:42:48,655 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742396_1572 (size=39749) 2024-11-23T05:42:48,745 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:48,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:48,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,745 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,745 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,746 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,898 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:48,898 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,898 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,899 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:48,906 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:48,907 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340628906, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:48,907 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:48,916 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T05:42:49,007 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:49,007 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 86 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340629005, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:49,007 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2149 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,008 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340629016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:49,018 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:49,018 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340629016, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:49,018 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2155 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:49,018 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2156 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,019 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:49,029 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:49,029 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 82 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340629026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:49,029 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 6, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 2167 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,030 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:49,050 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:49,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:49,050 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,051 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,051 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,055 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=239, memsize=42.5 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/448695501d6842c2ad142a0903d324db 2024-11-23T05:42:49,062 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/741a736fde314aff8379fad420a2e121 is 50, key is test_row_0/B:col10/1732340566861/Put/seqid=0 2024-11-23T05:42:49,068 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742397_1573 (size=12151) 2024-11-23T05:42:49,203 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,203 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,204 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,356 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,356 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,356 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,357 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,418 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:49,418 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340629416, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:49,418 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:49,426 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T05:42:49,468 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/741a736fde314aff8379fad420a2e121 2024-11-23T05:42:49,473 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/b78fdfe3202748b088c63e7c15028726 is 50, key is test_row_0/C:col10/1732340566861/Put/seqid=0 2024-11-23T05:42:49,476 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742398_1574 (size=12151) 2024-11-23T05:42:49,508 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,508 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,509 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,509 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,509 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,661 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:49,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:49,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,661 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,661 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,662 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,813 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,813 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,814 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] handler.RSProcedureHandler(58): pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,814 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=161 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,814 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=161 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:49,876 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=42.49 KB at sequenceid=239 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/b78fdfe3202748b088c63e7c15028726 2024-11-23T05:42:49,879 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/448695501d6842c2ad142a0903d324db as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/448695501d6842c2ad142a0903d324db 2024-11-23T05:42:49,882 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/448695501d6842c2ad142a0903d324db, entries=200, sequenceid=239, filesize=38.8 K 2024-11-23T05:42:49,886 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/741a736fde314aff8379fad420a2e121 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/741a736fde314aff8379fad420a2e121 2024-11-23T05:42:49,889 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/741a736fde314aff8379fad420a2e121, entries=150, sequenceid=239, filesize=11.9 K 2024-11-23T05:42:49,890 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/b78fdfe3202748b088c63e7c15028726 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/b78fdfe3202748b088c63e7c15028726 2024-11-23T05:42:49,892 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/b78fdfe3202748b088c63e7c15028726, entries=150, sequenceid=239, filesize=11.9 K 2024-11-23T05:42:49,893 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~127.47 KB/130530, heapSize ~334.69 KB/342720, currentSize=73.80 KB/75570 for f941d6128e412cb98f4108ef00511119 in 1655ms, sequenceid=239, compaction requested=false 2024-11-23T05:42:49,893 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:49,966 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=161 2024-11-23T05:42:49,966 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:49,966 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=73.80 KB heapSize=194.11 KB 2024-11-23T05:42:49,972 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e24f160a1c4d4a41b31b7516a3c8798e_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340568268/Put/seqid=0 2024-11-23T05:42:49,975 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742399_1575 (size=12304) 2024-11-23T05:42:50,375 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:50,378 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123e24f160a1c4d4a41b31b7516a3c8798e_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e24f160a1c4d4a41b31b7516a3c8798e_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:50,378 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/617603c573bc4bcd8809a80122ea59e8, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:50,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/617603c573bc4bcd8809a80122ea59e8 is 175, key is test_row_0/A:col10/1732340568268/Put/seqid=0 2024-11-23T05:42:50,389 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742400_1576 (size=31105) 2024-11-23T05:42:50,428 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:50,428 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:50,436 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T05:42:50,501 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:50,501 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 105 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340630497, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:50,501 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:50,609 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:50,609 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 106 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340630605, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:50,610 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:50,790 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=253, memsize=24.6 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/617603c573bc4bcd8809a80122ea59e8 2024-11-23T05:42:50,795 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d6caa1bf0e874adca70da0d608521a54 is 50, key is test_row_0/B:col10/1732340568268/Put/seqid=0 2024-11-23T05:42:50,798 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742401_1577 (size=12151) 2024-11-23T05:42:50,819 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:50,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 107 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340630816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:50,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 87 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:53990 deadline: 1732340631026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:51,028 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,028 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 85 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54026 deadline: 1732340631026, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:51,028 WARN [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_0' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4171 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:51,029 WARN [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_1' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4165 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-3 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,029 DEBUG [RPCClient-NioEventLoopGroup-4-1 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,039 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,039 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 84 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54076 deadline: 1732340631036, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:51,039 WARN [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4176 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,039 DEBUG [RPCClient-NioEventLoopGroup-4-10 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,049 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,049 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 83 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54046 deadline: 1732340631046, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:51,049 WARN [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRpcRetryingCaller(177): Call to 9abfc7c2569e,45719,1732340394120 for 'test_row_2' in f941d6128e412cb98f4108ef00511119 of TestAcidGuarantees failed, tries = 7, maxAttempts = 16, timeout = 1200000 ms, time elapsed = 4188 ms org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at jdk.internal.reflect.GeneratedConstructorAccessor40.newInstance(Unknown Source) ~[?:?] at jdk.internal.reflect.DelegatingConstructorAccessorImpl.newInstance(DelegatingConstructorAccessorImpl.java:45) ~[?:?] at java.lang.reflect.Constructor.newInstanceWithCaller(Constructor.java:499) ~[?:?] at java.lang.reflect.Constructor.newInstance(Constructor.java:480) ~[?:?] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.instantiateException(RemoteWithExtrasException.java:110) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RemoteWithExtrasException.unwrapRemoteException(RemoteWithExtrasException.java:100) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.ConnectionUtils.translateException(ConnectionUtils.java:219) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncRpcRetryingCaller.onError(AsyncRpcRetryingCaller.java:165) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.client.AsyncSingleRequestRpcRetryingCaller.lambda$call$4(AsyncSingleRequestRpcRetryingCaller.java:86) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.util.FutureUtils.lambda$addListener$0(FutureUtils.java:71) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at java.util.concurrent.CompletableFuture.uniWhenComplete(CompletableFuture.java:863) ~[?:?] at java.util.concurrent.CompletableFuture$UniWhenComplete.tryFire(CompletableFuture.java:841) ~[?:?] at java.util.concurrent.CompletableFuture.postComplete(CompletableFuture.java:510) ~[?:?] at java.util.concurrent.CompletableFuture.completeExceptionally(CompletableFuture.java:2162) ~[?:?] at org.apache.hadoop.hbase.client.ConnectionUtils$2.run(ConnectionUtils.java:625) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:56) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hbase.thirdparty.com.google.protobuf.RpcUtil$1.run(RpcUtil.java:47) ~[hbase-shaded-protobuf-4.1.9.jar:4.1.9] at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:397) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:430) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.AbstractRpcClient$3.run(AbstractRpcClient.java:425) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.callComplete(Call.java:117) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.Call.setException(Call.java:132) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcConnection.readResponse(RpcConnection.java:457) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.readResponse(NettyRpcDuplexHandler.java:125) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.NettyRpcDuplexHandler.channelRead(NettyRpcDuplexHandler.java:140) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:346) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:318) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:444) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.handler.timeout.IdleStateHandler.channelRead(IdleStateHandler.java:289) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:442) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:412) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1357) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:440) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:420) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:868) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:166) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:788) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeysOptimized(NioEventLoop.java:724) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:650) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:562) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.SingleThreadEventExecutor$4.run(SingleThreadEventExecutor.java:997) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.internal.ThreadExecutorMap$2.run(ThreadExecutorMap.java:74) ~[hbase-shaded-netty-4.1.9.jar:?] at org.apache.hbase.thirdparty.io.netty.util.concurrent.FastThreadLocalRunnable.run(FastThreadLocalRunnable.java:30) ~[hbase-shaded-netty-4.1.9.jar:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] Caused by: org.apache.hadoop.hbase.ipc.RemoteWithExtrasException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) at org.apache.hadoop.hbase.ipc.AbstractRpcClient.onCallFinished(AbstractRpcClient.java:392) ~[hbase-client-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] ... 32 more 2024-11-23T05:42:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,050 DEBUG [RPCClient-NioEventLoopGroup-4-7 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,129 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,130 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 108 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340631126, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,130 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,198 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d6caa1bf0e874adca70da0d608521a54 2024-11-23T05:42:51,203 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/52d6c015f1a840198c6764b5c818f592 is 50, key is test_row_0/C:col10/1732340568268/Put/seqid=0 2024-11-23T05:42:51,206 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742402_1578 (size=12151) 2024-11-23T05:42:51,607 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=24.60 KB at sequenceid=253 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/52d6c015f1a840198c6764b5c818f592 2024-11-23T05:42:51,610 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/617603c573bc4bcd8809a80122ea59e8 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/617603c573bc4bcd8809a80122ea59e8 2024-11-23T05:42:51,612 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/617603c573bc4bcd8809a80122ea59e8, entries=150, sequenceid=253, filesize=30.4 K 2024-11-23T05:42:51,613 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/d6caa1bf0e874adca70da0d608521a54 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d6caa1bf0e874adca70da0d608521a54 2024-11-23T05:42:51,616 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d6caa1bf0e874adca70da0d608521a54, entries=150, sequenceid=253, filesize=11.9 K 2024-11-23T05:42:51,616 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/52d6c015f1a840198c6764b5c818f592 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/52d6c015f1a840198c6764b5c818f592 2024-11-23T05:42:51,619 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/52d6c015f1a840198c6764b5c818f592, entries=150, sequenceid=253, filesize=11.9 K 2024-11-23T05:42:51,620 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(3140): Finished flush of dataSize ~73.80 KB/75570, heapSize ~194.06 KB/198720, currentSize=127.47 KB/130530 for f941d6128e412cb98f4108ef00511119 in 1654ms, sequenceid=253, compaction requested=true 2024-11-23T05:42:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:51,620 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=161}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=161 2024-11-23T05:42:51,620 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=161 2024-11-23T05:42:51,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=161, resume processing ppid=160 2024-11-23T05:42:51,622 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=161, ppid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 3.3330 sec 2024-11-23T05:42:51,623 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=160, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=160, table=TestAcidGuarantees in 3.3360 sec 2024-11-23T05:42:51,637 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:51,637 DEBUG [MemStoreFlusher.0 {}] regionserver.FlushAllLargeStoresPolicy(69): Since none of the CFs were above the size, flushing all. 2024-11-23T05:42:51,637 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=134.18 KB heapSize=352.31 KB 2024-11-23T05:42:51,642 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112382f1cdccdbf0401ebc012565d981dc73_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340570483/Put/seqid=0 2024-11-23T05:42:51,645 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742403_1579 (size=14994) 2024-11-23T05:42:51,674 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,675 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 120 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340631671, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,675 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,786 WARN [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,786 DEBUG [RpcServer.default.FPBQ.Fifo.handler=0,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 121 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340631786, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:51,787 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:51,999 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:51,999 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 122 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340631996, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:52,000 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:52,046 DEBUG [MemStoreFlusher.0 {}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:52,048 INFO [MemStoreFlusher.0 {}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e2024112382f1cdccdbf0401ebc012565d981dc73_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112382f1cdccdbf0401ebc012565d981dc73_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:52,049 DEBUG [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/dab1066c049c4154ab72339bf78a3080, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:52,049 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/dab1066c049c4154ab72339bf78a3080 is 175, key is test_row_0/A:col10/1732340570483/Put/seqid=0 2024-11-23T05:42:52,052 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742404_1580 (size=39949) 2024-11-23T05:42:52,308 WARN [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:52,308 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 123 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340632305, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:52,308 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:52,347 DEBUG [FsDatasetAsyncDiskServiceFixer {}] hbase.HBaseTestingUtil$FsDatasetAsyncDiskServiceFixer(597): NoSuchFieldException: threadGroup; It might because your Hadoop version > 3.2.3 or 3.3.4, See HBASE-27595 for details. 2024-11-23T05:42:52,452 INFO [MemStoreFlusher.0 {}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=276, memsize=44.7 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/dab1066c049c4154ab72339bf78a3080 2024-11-23T05:42:52,456 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=160 2024-11-23T05:42:52,457 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:52,457 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$22(4506): Client=jenkins//172.17.0.2 flush TestAcidGuarantees 2024-11-23T05:42:52,457 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/01db9728e53846c3bd3ed8151980a9e3 is 50, key is test_row_0/B:col10/1732340570483/Put/seqid=0 2024-11-23T05:42:52,458 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees 2024-11-23T05:42:52,459 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T05:42:52,459 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_PREPARE, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_PREPARE 2024-11-23T05:42:52,460 INFO [PEWorker-3 {}] procedure.FlushTableProcedure(91): pid=162, state=RUNNABLE:FLUSH_TABLE_FLUSH_REGIONS, hasLock=true; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees execute state=FLUSH_TABLE_FLUSH_REGIONS 2024-11-23T05:42:52,460 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=163, ppid=162, state=RUNNABLE, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure}] 2024-11-23T05:42:52,460 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742405_1581 (size=12301) 2024-11-23T05:42:52,566 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T05:42:52,612 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T05:42:52,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:52,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:52,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:52,612 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,612 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,613 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,764 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T05:42:52,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:52,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:52,764 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:52,765 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,765 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,765 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T05:42:52,819 WARN [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(5310): Region is too busy due to exceeding memstore size limit. org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) ~[classes/:?] at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) ~[hbase-common-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) ~[classes/:?] at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] 2024-11-23T05:42:52,819 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] ipc.CallRunner(138): callId: 124 service: ClientService methodName: Mutate size: 4.7 K connection: 172.17.0.2:54012 deadline: 1732340632816, exception=org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(64): Try updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 , the old value is region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5, error=org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(72): The actual exception when updating region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 is org.apache.hadoop.hbase.RegionTooBusyException: org.apache.hadoop.hbase.RegionTooBusyException: Over memstore limit=512.0 K, regionName=f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 at org.apache.hadoop.hbase.regionserver.HRegion.checkResources(HRegion.java:5308) at org.apache.hadoop.hbase.regionserver.HRegion.lambda$put$11(HRegion.java:3329) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:216) at org.apache.hadoop.hbase.regionserver.HRegion.put(HRegion.java:3322) at org.apache.hadoop.hbase.regionserver.RSRpcServices.put(RSRpcServices.java:3018) at org.apache.hadoop.hbase.regionserver.RSRpcServices.mutate(RSRpcServices.java:2981) at org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos$ClientService$2.callBlockingMethod(ClientProtos.java:43506) at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) 2024-11-23T05:42:52,819 DEBUG [RPCClient-NioEventLoopGroup-4-5 {}] client.AsyncRegionLocatorHelper(76): Will not update region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., hostname=9abfc7c2569e,45719,1732340394120, seqNum=5 because the exception is null or not the one we care about 2024-11-23T05:42:52,861 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/01db9728e53846c3bd3ed8151980a9e3 2024-11-23T05:42:52,866 DEBUG [MemStoreFlusher.0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/26d572fcc14d430e8524559903ddcb80 is 50, key is test_row_0/C:col10/1732340570483/Put/seqid=0 2024-11-23T05:42:52,868 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742406_1582 (size=12301) 2024-11-23T05:42:52,917 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T05:42:52,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:52,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:52,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:52,917 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,917 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:52,918 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:53,069 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T05:42:53,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:53,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,069 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:53,069 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-0 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:53,070 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:53,086 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T05:42:53,222 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=1,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T05:42:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,222 ERROR [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] handler.RSProcedureHandler(58): pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:53,222 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-1 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(58): Failed to complete execution of pid=163 java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:53,222 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4177): Remote procedure failed, pid=163 org.apache.hadoop.hbase.procedure2.RemoteProcedureException: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.procedure2.RemoteProcedureException.fromProto(RemoteProcedureException.java:123) ~[hbase-procedure-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.master.MasterRpcServices.lambda$reportProcedureDone$5(MasterRpcServices.java:2580) ~[classes/:?] at java.util.ArrayList.forEach(ArrayList.java:1511) ~[?:?] at java.util.Collections$UnmodifiableCollection.forEach(Collections.java:1092) ~[?:?] at org.apache.hadoop.hbase.master.MasterRpcServices.reportProcedureDone(MasterRpcServices.java:2575) ~[classes/:?] at org.apache.hadoop.hbase.shaded.protobuf.generated.RegionServerStatusProtos$RegionServerStatusService$2.callBlockingMethod(RegionServerStatusProtos.java:16726) ~[hbase-protocol-shaded-4.0.0-alpha-1-SNAPSHOT.jar:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcServer.call(RpcServer.java:444) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.CallRunner.run(CallRunner.java:124) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:102) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.ipc.RpcHandler.run(RpcHandler.java:82) ~[classes/:4.0.0-alpha-1-SNAPSHOT] Caused by: java.io.IOException: Unable to complete flush {ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''} at org.apache.hadoop.hbase.regionserver.FlushRegionCallable.doCall(FlushRegionCallable.java:61) ~[classes/:?] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:35) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.procedure2.BaseRSProcedureCallable.call(BaseRSProcedureCallable.java:23) ~[classes/:4.0.0-alpha-1-SNAPSHOT] at org.apache.hadoop.hbase.regionserver.handler.RSProcedureHandler.process(RSProcedureHandler.java:56) ~[classes/:?] at org.apache.hadoop.hbase.executor.EventHandler.run(EventHandler.java:104) ~[classes/:?] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1136) ~[?:?] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:635) ~[?:?] at java.lang.Thread.run(Thread.java:840) ~[?:?] 2024-11-23T05:42:53,269 INFO [MemStoreFlusher.0 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=44.73 KB at sequenceid=276 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/26d572fcc14d430e8524559903ddcb80 2024-11-23T05:42:53,272 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/dab1066c049c4154ab72339bf78a3080 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/dab1066c049c4154ab72339bf78a3080 2024-11-23T05:42:53,275 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/dab1066c049c4154ab72339bf78a3080, entries=200, sequenceid=276, filesize=39.0 K 2024-11-23T05:42:53,275 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/01db9728e53846c3bd3ed8151980a9e3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/01db9728e53846c3bd3ed8151980a9e3 2024-11-23T05:42:53,278 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/01db9728e53846c3bd3ed8151980a9e3, entries=150, sequenceid=276, filesize=12.0 K 2024-11-23T05:42:53,278 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/26d572fcc14d430e8524559903ddcb80 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/26d572fcc14d430e8524559903ddcb80 2024-11-23T05:42:53,281 INFO [MemStoreFlusher.0 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/26d572fcc14d430e8524559903ddcb80, entries=150, sequenceid=276, filesize=12.0 K 2024-11-23T05:42:53,282 INFO [MemStoreFlusher.0 {}] regionserver.HRegion(3140): Finished flush of dataSize ~134.18 KB/137400, heapSize ~352.27 KB/360720, currentSize=67.09 KB/68700 for f941d6128e412cb98f4108ef00511119 in 1644ms, sequenceid=276, compaction requested=true 2024-11-23T05:42:53,282 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:53,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:A, priority=-2147483648, current under compaction store size is 1 2024-11-23T05:42:53,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:53,282 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:53,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:B, priority=-2147483648, current under compaction store size is 2 2024-11-23T05:42:53,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:53,282 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:53,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(403): Add compact mark for store f941d6128e412cb98f4108ef00511119:C, priority=-2147483648, current under compaction store size is 3 2024-11-23T05:42:53,282 DEBUG [MemStoreFlusher.0 {}] regionserver.CompactSplit(411): Small Compaction requested: system; Because: MemStoreFlusher.0; compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:53,282 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 142386 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:53,282 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/B is initiating minor compaction (all files) 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/A is initiating minor compaction (all files) 2024-11-23T05:42:53,283 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/B in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,283 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/A in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,283 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/d4e0af6a317841d8bbae2c4b68db6836, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/448695501d6842c2ad142a0903d324db, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/617603c573bc4bcd8809a80122ea59e8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/dab1066c049c4154ab72339bf78a3080] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=139.0 K 2024-11-23T05:42:53,283 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/97ce7dbb38834f98bc3123df0f9dd013, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/741a736fde314aff8379fad420a2e121, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d6caa1bf0e874adca70da0d608521a54, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/01db9728e53846c3bd3ed8151980a9e3] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=48.1 K 2024-11-23T05:42:53,283 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(181): MOB compaction: major=false isAll=true priority=12 throughput controller=DefaultCompactionThroughputController [maxThroughput=53.85 MB/second, activeCompactions=0] table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(191): MOB compaction table=TestAcidGuarantees cf=A region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. files: [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/d4e0af6a317841d8bbae2c4b68db6836, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/448695501d6842c2ad142a0903d324db, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/617603c573bc4bcd8809a80122ea59e8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/dab1066c049c4154ab72339bf78a3080] 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 97ce7dbb38834f98bc3123df0f9dd013, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732340566206 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting d4e0af6a317841d8bbae2c4b68db6836, keycount=150, bloomtype=ROW, size=30.8 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732340566206 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 741a736fde314aff8379fad420a2e121, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732340566861 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 448695501d6842c2ad142a0903d324db, keycount=200, bloomtype=ROW, size=38.8 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732340566855 2024-11-23T05:42:53,283 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting d6caa1bf0e874adca70da0d608521a54, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732340568261 2024-11-23T05:42:53,284 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting 617603c573bc4bcd8809a80122ea59e8, keycount=150, bloomtype=ROW, size=30.4 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732340568261 2024-11-23T05:42:53,284 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 01db9728e53846c3bd3ed8151980a9e3, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340570460 2024-11-23T05:42:53,284 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] compactions.Compactor(225): Compacting dab1066c049c4154ab72339bf78a3080, keycount=200, bloomtype=ROW, size=39.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340570460 2024-11-23T05:42:53,288 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(321): Compact MOB=false optimized configured=false optimized enabled=false maximum MOB file size=1073741824 major=true store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:53,291 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#B#compaction#489 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 1 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:53,291 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8a0c53a162ab4a4c82b9c3cb3920881f is 50, key is test_row_0/B:col10/1732340570483/Put/seqid=0 2024-11-23T05:42:53,293 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(635): New MOB writer created=d41d8cd98f00b204e9800998ecf8427e20241123bb84d154373e43d1823f99f1db796883_f941d6128e412cb98f4108ef00511119 store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:53,295 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(656): Commit or abort size=0 mobCells=0 major=true file=d41d8cd98f00b204e9800998ecf8427e20241123bb84d154373e43d1823f99f1db796883_f941d6128e412cb98f4108ef00511119, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:53,295 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] mob.DefaultMobStoreCompactor(667): Aborting writer for hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123bb84d154373e43d1823f99f1db796883_f941d6128e412cb98f4108ef00511119 because there are no MOB cells, store=[table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:53,299 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742408_1584 (size=4469) 2024-11-23T05:42:53,300 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742407_1583 (size=12915) 2024-11-23T05:42:53,300 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#A#compaction#488 average throughput is 2.04 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:53,300 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0d3a70c4ef2944869cc7e2900400aa4b is 175, key is test_row_0/A:col10/1732340570483/Put/seqid=0 2024-11-23T05:42:53,304 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8a0c53a162ab4a4c82b9c3cb3920881f as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8a0c53a162ab4a4c82b9c3cb3920881f 2024-11-23T05:42:53,311 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/B of f941d6128e412cb98f4108ef00511119 into 8a0c53a162ab4a4c82b9c3cb3920881f(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:53,311 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:53,311 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/B, priority=12, startTime=1732340573282; duration=0sec 2024-11-23T05:42:53,311 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=1), splitQueue=0 2024-11-23T05:42:53,311 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:B 2024-11-23T05:42:53,311 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.SortedCompactionPolicy(75): Selecting compaction from 4 store files, 0 compacting, 4 eligible, 16 blocking 2024-11-23T05:42:53,313 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.ExploringCompactionPolicy(116): Exploring compaction algorithm has selected 4 files of size 49232 starting at candidate #0 after considering 3 permutations with 3 in ratio 2024-11-23T05:42:53,313 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1541): f941d6128e412cb98f4108ef00511119/C is initiating minor compaction (all files) 2024-11-23T05:42:53,313 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2416): Starting compaction of f941d6128e412cb98f4108ef00511119/C in TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,313 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1176): Starting compaction of [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/642b339d2548496e8a436d83de2c5a05, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/b78fdfe3202748b088c63e7c15028726, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/52d6c015f1a840198c6764b5c818f592, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/26d572fcc14d430e8524559903ddcb80] into tmpdir=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp, totalSize=48.1 K 2024-11-23T05:42:53,314 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 642b339d2548496e8a436d83de2c5a05, keycount=150, bloomtype=ROW, size=12.3 K, encoding=NONE, compression=NONE, seqNum=214, earliestPutTs=1732340566206 2024-11-23T05:42:53,314 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting b78fdfe3202748b088c63e7c15028726, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=239, earliestPutTs=1732340566861 2024-11-23T05:42:53,314 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 52d6c015f1a840198c6764b5c818f592, keycount=150, bloomtype=ROW, size=11.9 K, encoding=NONE, compression=NONE, seqNum=253, earliestPutTs=1732340568261 2024-11-23T05:42:53,314 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] compactions.Compactor(225): Compacting 26d572fcc14d430e8524559903ddcb80, keycount=150, bloomtype=ROW, size=12.0 K, encoding=NONE, compression=NONE, seqNum=276, earliestPutTs=1732340570460 2024-11-23T05:42:53,330 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742409_1585 (size=31869) 2024-11-23T05:42:53,331 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] throttle.PressureAwareThroughputController(145): f941d6128e412cb98f4108ef00511119#C#compaction#490 average throughput is 6.55 MB/second, slept 0 time(s) and total slept time is 0 ms. 0 active operations remaining, total limit is 53.85 MB/second 2024-11-23T05:42:53,332 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/ede7fbb3181a42cc88f6d1209e80e3c7 is 50, key is test_row_0/C:col10/1732340570483/Put/seqid=0 2024-11-23T05:42:53,334 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/0d3a70c4ef2944869cc7e2900400aa4b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d3a70c4ef2944869cc7e2900400aa4b 2024-11-23T05:42:53,336 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742410_1586 (size=12915) 2024-11-23T05:42:53,339 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/A of f941d6128e412cb98f4108ef00511119 into 0d3a70c4ef2944869cc7e2900400aa4b(size=31.1 K), total size for store is 31.1 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:53,339 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:53,339 INFO [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/A, priority=12, startTime=1732340573282; duration=0sec 2024-11-23T05:42:53,339 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:53,339 DEBUG [RS:0;9abfc7c2569e:45719-shortCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:A 2024-11-23T05:42:53,340 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/ede7fbb3181a42cc88f6d1209e80e3c7 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/ede7fbb3181a42cc88f6d1209e80e3c7 2024-11-23T05:42:53,343 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HStore(1337): Completed compaction of 4 (all) file(s) in f941d6128e412cb98f4108ef00511119/C of f941d6128e412cb98f4108ef00511119 into ede7fbb3181a42cc88f6d1209e80e3c7(size=12.6 K), total size for store is 12.6 K. This selection was in queue for 0sec, and took 0sec to execute. 2024-11-23T05:42:53,343 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.HRegion(2446): Compaction status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:53,343 INFO [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(669): Completed compaction region=TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., storeName=f941d6128e412cb98f4108ef00511119/C, priority=12, startTime=1732340573282; duration=0sec 2024-11-23T05:42:53,343 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(701): Status compactionQueue=(longCompactions=0:shortCompactions=0), splitQueue=0 2024-11-23T05:42:53,343 DEBUG [RS:0;9abfc7c2569e:45719-longCompactions-0 {}] regionserver.CompactSplit$CompactionRunner(719): Remove under compaction mark for store: f941d6128e412cb98f4108ef00511119:C 2024-11-23T05:42:53,374 DEBUG [RpcServer.priority.RWQ.Fifo.read.handler=2,queue=1,port=45719 {}] regionserver.RSRpcServices(3929): Executing remote procedure class org.apache.hadoop.hbase.regionserver.FlushRegionCallable, pid=163 2024-11-23T05:42:53,374 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(51): Starting region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:53,374 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=67.09 KB heapSize=176.53 KB 2024-11-23T05:42:53,379 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d7bb7eb440a34df1b9c5286586a25078_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340571646/Put/seqid=0 2024-11-23T05:42:53,381 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742411_1587 (size=12454) 2024-11-23T05:42:53,511 INFO [Thread-2302 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2302. 2024-11-23T05:42:53,511 INFO [Thread-2306 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2306. 2024-11-23T05:42:53,511 DEBUG [Thread-2302 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:53,511 DEBUG [Thread-2306 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:53,511 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,511 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,511 INFO [Thread-2310 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2310. 2024-11-23T05:42:53,511 DEBUG [Thread-2310 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:53,511 DEBUG [Thread-2306 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,511 DEBUG [Thread-2302 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,511 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:53,511 DEBUG [Thread-2310 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:53,511 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:53,512 INFO [Thread-2308 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2308. 2024-11-23T05:42:53,512 DEBUG [Thread-2308 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:53,512 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,512 DEBUG [Thread-2308 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,512 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:53,513 INFO [Thread-2304 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2304. 2024-11-23T05:42:53,513 DEBUG [Thread-2304 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicGetReader.workDone(AcidGuaranteesTestTool.java:234) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:53,513 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,513 DEBUG [Thread-2304 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,513 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:53,596 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T05:42:53,783 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:53,791 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123d7bb7eb440a34df1b9c5286586a25078_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d7bb7eb440a34df1b9c5286586a25078_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:53,793 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/bdbcd9613cdc4a00b00c2a32295dbf7a, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:53,794 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/bdbcd9613cdc4a00b00c2a32295dbf7a is 175, key is test_row_0/A:col10/1732340571646/Put/seqid=0 2024-11-23T05:42:53,799 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742412_1588 (size=31255) 2024-11-23T05:42:53,839 DEBUG [RpcServer.default.FPBQ.Fifo.handler=2,queue=0,port=45719 {}] regionserver.HRegion(8855): Flush requested on f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:53,839 DEBUG [MemStoreFlusher.0 {}] regionserver.HRegion(2561): NOT flushing TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. as already flushing 2024-11-23T05:42:53,840 INFO [Thread-2299 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2299. 2024-11-23T05:42:53,840 DEBUG [Thread-2299 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:53,840 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,840 DEBUG [Thread-2299 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:53,840 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:54,200 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=292, memsize=22.4 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/bdbcd9613cdc4a00b00c2a32295dbf7a 2024-11-23T05:42:54,211 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/78d4a77c22784d9fa4b73de78636fc07 is 50, key is test_row_0/B:col10/1732340571646/Put/seqid=0 2024-11-23T05:42:54,216 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742413_1589 (size=12301) 2024-11-23T05:42:54,606 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T05:42:54,618 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/78d4a77c22784d9fa4b73de78636fc07 2024-11-23T05:42:54,633 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/7f888b1892c144778c49e1a09bde6271 is 50, key is test_row_0/C:col10/1732340571646/Put/seqid=0 2024-11-23T05:42:54,636 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742414_1590 (size=12301) 2024-11-23T05:42:55,038 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=22.36 KB at sequenceid=292 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/7f888b1892c144778c49e1a09bde6271 2024-11-23T05:42:55,040 INFO [Thread-2293 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2293. 2024-11-23T05:42:55,041 DEBUG [Thread-2293 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:55,041 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,041 DEBUG [Thread-2293 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,041 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:55,046 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/bdbcd9613cdc4a00b00c2a32295dbf7a as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/bdbcd9613cdc4a00b00c2a32295dbf7a 2024-11-23T05:42:55,050 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/bdbcd9613cdc4a00b00c2a32295dbf7a, entries=150, sequenceid=292, filesize=30.5 K 2024-11-23T05:42:55,051 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/78d4a77c22784d9fa4b73de78636fc07 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/78d4a77c22784d9fa4b73de78636fc07 2024-11-23T05:42:55,053 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/78d4a77c22784d9fa4b73de78636fc07, entries=150, sequenceid=292, filesize=12.0 K 2024-11-23T05:42:55,054 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/7f888b1892c144778c49e1a09bde6271 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/7f888b1892c144778c49e1a09bde6271 2024-11-23T05:42:55,056 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/7f888b1892c144778c49e1a09bde6271, entries=150, sequenceid=292, filesize=12.0 K 2024-11-23T05:42:55,056 INFO [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(3140): Finished flush of dataSize ~67.09 KB/68700, heapSize ~176.48 KB/180720, currentSize=13.42 KB/13740 for f941d6128e412cb98f4108ef00511119 in 1682ms, sequenceid=292, compaction requested=false 2024-11-23T05:42:55,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.HRegion(2603): Flush status journal for f941d6128e412cb98f4108ef00511119: 2024-11-23T05:42:55,056 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.FlushRegionCallable(64): Closing region operation on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:55,057 DEBUG [RS_FLUSH_OPERATIONS-regionserver/9abfc7c2569e:0-2 {event_type=RS_FLUSH_REGIONS, pid=163}] regionserver.RemoteProcedureResultReporter(62): Successfully complete execution of pid=163 2024-11-23T05:42:55,057 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster(4169): Remote procedure done, pid=163 2024-11-23T05:42:55,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=163, resume processing ppid=162 2024-11-23T05:42:55,059 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=163, ppid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushRegionProcedure in 2.5980 sec 2024-11-23T05:42:55,060 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=162, state=SUCCESS, hasLock=false; org.apache.hadoop.hbase.master.procedure.FlushTableProcedure, id=162, table=TestAcidGuarantees in 2.6020 sec 2024-11-23T05:42:55,078 INFO [Thread-2291 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2291. 2024-11-23T05:42:55,078 DEBUG [Thread-2291 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:55,078 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,078 DEBUG [Thread-2291 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,078 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:55,088 INFO [Thread-2295 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2295. 2024-11-23T05:42:55,088 INFO [Thread-2297 {}] client.AsyncConnectionImpl(233): Connection has been closed by Thread-2297. 2024-11-23T05:42:55,088 DEBUG [Thread-2295 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:55,088 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,088 DEBUG [Thread-2297 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool$AtomicityWriter.workDone(AcidGuaranteesTestTool.java:178) at org.apache.hadoop.hbase.MultithreadedTestUtil$RepeatingTestThread.doWork(MultithreadedTestUtil.java:152) at org.apache.hadoop.hbase.MultithreadedTestUtil$TestThread.run(MultithreadedTestUtil.java:123) 2024-11-23T05:42:55,088 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,088 DEBUG [Thread-2295 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,088 DEBUG [Thread-2297 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:55,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:55,088 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:56,617 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=162 2024-11-23T05:42:56,617 INFO [RPCClient-NioEventLoopGroup-4-12 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: FLUSH, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(392): Finished test. Writers: 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 42 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 40 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 39 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 38 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(394): wrote 74 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(396): Readers: 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4215 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4022 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4141 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4214 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(398): read 4016 2024-11-23T05:42:56,618 INFO [Time-limited test {}] hbase.AcidGuaranteesTestTool(400): Scanners: 2024-11-23T05:42:56,619 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:42:56,619 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.client.ConnectionOverAsyncConnection.close(ConnectionOverAsyncConnection.java:134) at org.apache.hadoop.hbase.AcidGuaranteesTestTool.doWork(AcidGuaranteesTestTool.java:131) at org.apache.hadoop.hbase.util.AbstractHBaseTool.run(AbstractHBaseTool.java:150) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.runTestAtomicity(AcidGuaranteesTestBase.java:102) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.testMobGetAtomicity(AcidGuaranteesTestBase.java:122) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.runners.ParentRunner$3.evaluate(ParentRunner.java:306) at org.junit.runners.BlockJUnit4ClassRunner$1.evaluate(BlockJUnit4ClassRunner.java:100) at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:366) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:103) at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:63) at org.junit.runners.ParentRunner$4.run(ParentRunner.java:331) at org.junit.runners.ParentRunner$1.schedule(ParentRunner.java:79) at org.junit.runners.ParentRunner.runChildren(ParentRunner.java:329) at org.junit.runners.ParentRunner.access$100(ParentRunner.java:66) at org.junit.runners.ParentRunner$2.evaluate(ParentRunner.java:293) at org.junit.internal.runners.statements.RunBefores.evaluate(RunBefores.java:26) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:27) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:42:56,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:56,619 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:56,620 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:56,621 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$13(2820): Client=jenkins//172.17.0.2 disable TestAcidGuarantees 2024-11-23T05:42:56,623 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=164, state=RUNNABLE:DISABLE_TABLE_PREPARE, hasLock=false; DisableTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:56,625 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T05:42:56,626 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340576626"}]},"ts":"1732340576626"} 2024-11-23T05:42:56,628 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLING in hbase:meta 2024-11-23T05:42:56,628 INFO [PEWorker-5 {}] procedure.DisableTableProcedure(284): Set TestAcidGuarantees to state=DISABLING 2024-11-23T05:42:56,628 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=165, ppid=164, state=RUNNABLE:CLOSE_TABLE_REGIONS_SCHEDULE, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees}] 2024-11-23T05:42:56,630 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, UNASSIGN}] 2024-11-23T05:42:56,631 INFO [PEWorker-1 {}] procedure.MasterProcedureScheduler(851): Took xlock for pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, UNASSIGN 2024-11-23T05:42:56,632 INFO [PEWorker-1 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=CLOSING, regionLocation=9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:56,633 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=166, ppid=165, state=RUNNABLE:REGION_STATE_TRANSITION_CLOSE, hasLock=true; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, UNASSIGN because future has completed 2024-11-23T05:42:56,634 DEBUG [PEWorker-4 {}] assignment.TransitRegionStateProcedure(375): Close region: isSplit: false: evictOnSplit: true: evictOnClose: false 2024-11-23T05:42:56,634 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1860): Initialized subprocedures=[{pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120}] 2024-11-23T05:42:56,736 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T05:42:56,787 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(122): Close f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:56,787 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(136): Unassign region: split region: false: evictCache: false 2024-11-23T05:42:56,788 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1722): Closing f941d6128e412cb98f4108ef00511119, disabling compactions & flushes 2024-11-23T05:42:56,788 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1755): Closing region TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:56,788 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1776): Time limited wait for close lock on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:56,788 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1843): Acquired close lock on TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. after waiting 0 ms 2024-11-23T05:42:56,788 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1853): Updates disabled for region TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:56,788 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(2902): Flushing f941d6128e412cb98f4108ef00511119 3/3 column families, dataSize=33.54 KB heapSize=88.64 KB 2024-11-23T05:42:56,798 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123bc8b51562e0a4d50b867aeee1d2f76b4_f941d6128e412cb98f4108ef00511119 is 50, key is test_row_0/A:col10/1732340575077/Put/seqid=0 2024-11-23T05:42:56,802 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742415_1591 (size=12454) 2024-11-23T05:42:56,947 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T05:42:57,203 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] storefiletracker.StoreFileTrackerFactory(122): instantiating StoreFileTracker impl org.apache.hadoop.hbase.regionserver.storefiletracker.DefaultStoreFileTracker 2024-11-23T05:42:57,209 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HMobStore(268): FLUSH Renaming flushed file from hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/.tmp/d41d8cd98f00b204e9800998ecf8427e20241123bc8b51562e0a4d50b867aeee1d2f76b4_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123bc8b51562e0a4d50b867aeee1d2f76b4_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:57,210 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(263): Flush store file: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/387ac5ed0e3749958a1116ac8598f04b, store: [table=TestAcidGuarantees family=A region=f941d6128e412cb98f4108ef00511119] 2024-11-23T05:42:57,211 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/387ac5ed0e3749958a1116ac8598f04b is 175, key is test_row_0/A:col10/1732340575077/Put/seqid=0 2024-11-23T05:42:57,217 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742416_1592 (size=31255) 2024-11-23T05:42:57,256 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T05:42:57,286 DEBUG [regionserver/9abfc7c2569e:0.Chore.1 {}] throttle.PressureAwareCompactionThroughputController(103): CompactionPressure is 0.0, tune throughput to 50.00 MB/second 2024-11-23T05:42:57,619 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] mob.DefaultMobStoreFlusher(147): Mob store is flushed, sequenceid=300, memsize=11.2 K, hasBloomFilter=true, into tmp file hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/387ac5ed0e3749958a1116ac8598f04b 2024-11-23T05:42:57,631 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8a3b406546f74948bcc2461c7c0c0392 is 50, key is test_row_0/B:col10/1732340575077/Put/seqid=0 2024-11-23T05:42:57,635 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742417_1593 (size=12301) 2024-11-23T05:42:57,766 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T05:42:58,037 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8a3b406546f74948bcc2461c7c0c0392 2024-11-23T05:42:58,050 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/e193a15fe0f744338866cc5cf293e64c is 50, key is test_row_0/C:col10/1732340575077/Put/seqid=0 2024-11-23T05:42:58,054 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742418_1594 (size=12301) 2024-11-23T05:42:58,456 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=11.18 KB at sequenceid=300 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/e193a15fe0f744338866cc5cf293e64c 2024-11-23T05:42:58,465 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/A/387ac5ed0e3749958a1116ac8598f04b as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/387ac5ed0e3749958a1116ac8598f04b 2024-11-23T05:42:58,469 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/387ac5ed0e3749958a1116ac8598f04b, entries=150, sequenceid=300, filesize=30.5 K 2024-11-23T05:42:58,470 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/B/8a3b406546f74948bcc2461c7c0c0392 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8a3b406546f74948bcc2461c7c0c0392 2024-11-23T05:42:58,473 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8a3b406546f74948bcc2461c7c0c0392, entries=150, sequenceid=300, filesize=12.0 K 2024-11-23T05:42:58,474 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/.tmp/C/e193a15fe0f744338866cc5cf293e64c as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/e193a15fe0f744338866cc5cf293e64c 2024-11-23T05:42:58,478 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/e193a15fe0f744338866cc5cf293e64c, entries=150, sequenceid=300, filesize=12.0 K 2024-11-23T05:42:58,479 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(3140): Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f941d6128e412cb98f4108ef00511119 in 1691ms, sequenceid=300, compaction requested=true 2024-11-23T05:42:58,479 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d1ad5e8235b4a94a5b5fdd0d79776b7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3a91b1c932da40f9ae524240ac8f4835, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3ae69c16b7c8418da873e6a32aa06b91, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/ae11febd10c14172a4160aacbd33886c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/254035676be749728a357d1d70d43294, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/914ac9d5864a47f5be2a326a1c833e3e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a2f4e920490b45e2999e147bd1f1c155, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0ea8c96e9f09450bae9300c70c8b318f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/95634794404f45d4aa7276e132bb62c6, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/06d9004ac38142938185864cc3ff1061, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/9d461d7d5405466b99039fb11480c36d, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/50d05d5baeec4d68b4c06bdb7c06e6ba, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/2bd4547f122242d082d34c42803089ef, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/d4e0af6a317841d8bbae2c4b68db6836, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a0734cd7fdda4199bc61d4dc856af572, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/448695501d6842c2ad142a0903d324db, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/617603c573bc4bcd8809a80122ea59e8, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/dab1066c049c4154ab72339bf78a3080] to archive 2024-11-23T05:42:58,480 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:42:58,481 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d1ad5e8235b4a94a5b5fdd0d79776b7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d1ad5e8235b4a94a5b5fdd0d79776b7 2024-11-23T05:42:58,482 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3a91b1c932da40f9ae524240ac8f4835 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3a91b1c932da40f9ae524240ac8f4835 2024-11-23T05:42:58,484 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3ae69c16b7c8418da873e6a32aa06b91 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/3ae69c16b7c8418da873e6a32aa06b91 2024-11-23T05:42:58,485 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/ae11febd10c14172a4160aacbd33886c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/ae11febd10c14172a4160aacbd33886c 2024-11-23T05:42:58,487 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/254035676be749728a357d1d70d43294 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/254035676be749728a357d1d70d43294 2024-11-23T05:42:58,488 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/914ac9d5864a47f5be2a326a1c833e3e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/914ac9d5864a47f5be2a326a1c833e3e 2024-11-23T05:42:58,490 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a2f4e920490b45e2999e147bd1f1c155 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a2f4e920490b45e2999e147bd1f1c155 2024-11-23T05:42:58,491 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0ea8c96e9f09450bae9300c70c8b318f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0ea8c96e9f09450bae9300c70c8b318f 2024-11-23T05:42:58,492 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/95634794404f45d4aa7276e132bb62c6 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/95634794404f45d4aa7276e132bb62c6 2024-11-23T05:42:58,494 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/06d9004ac38142938185864cc3ff1061 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/06d9004ac38142938185864cc3ff1061 2024-11-23T05:42:58,495 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/9d461d7d5405466b99039fb11480c36d to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/9d461d7d5405466b99039fb11480c36d 2024-11-23T05:42:58,496 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/50d05d5baeec4d68b4c06bdb7c06e6ba to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/50d05d5baeec4d68b4c06bdb7c06e6ba 2024-11-23T05:42:58,497 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/2bd4547f122242d082d34c42803089ef to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/2bd4547f122242d082d34c42803089ef 2024-11-23T05:42:58,499 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/d4e0af6a317841d8bbae2c4b68db6836 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/d4e0af6a317841d8bbae2c4b68db6836 2024-11-23T05:42:58,500 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a0734cd7fdda4199bc61d4dc856af572 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/a0734cd7fdda4199bc61d4dc856af572 2024-11-23T05:42:58,501 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/448695501d6842c2ad142a0903d324db to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/448695501d6842c2ad142a0903d324db 2024-11-23T05:42:58,503 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/617603c573bc4bcd8809a80122ea59e8 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/617603c573bc4bcd8809a80122ea59e8 2024-11-23T05:42:58,504 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/dab1066c049c4154ab72339bf78a3080 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/dab1066c049c4154ab72339bf78a3080 2024-11-23T05:42:58,506 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/15aa5550c1164693802b7959df6794f3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/6bb3ae26edc948f7bcd4aded0bb7a0ab, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/805e1c6ea2ea499a97b0d8538d5b72ba, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/264c80e833cb46a8942b5026c3db4e1b, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d54528007c3649e1b59d9961fb33f614, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/66ec397f737941728639d28676188a66, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/ee87403090484184a5018345a997696e, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8e95f8daff5940c1aa2f0495133beca7, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d89b7efb7ead441a8632728d097fece3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/5e8a39ac29944a2aacaadac3377a0f29, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/b85b63488771425a8418cd978fb063f3, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3b1e0035440a491cb316de3afc1a39cc, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8dca9b0a0a5a482eaa79426502e8c42c, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/97ce7dbb38834f98bc3123df0f9dd013, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3aba7490273b4a1fb774881fd56326ad, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/741a736fde314aff8379fad420a2e121, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d6caa1bf0e874adca70da0d608521a54, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/01db9728e53846c3bd3ed8151980a9e3] to archive 2024-11-23T05:42:58,507 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:42:58,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/15aa5550c1164693802b7959df6794f3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/15aa5550c1164693802b7959df6794f3 2024-11-23T05:42:58,508 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/6bb3ae26edc948f7bcd4aded0bb7a0ab to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/6bb3ae26edc948f7bcd4aded0bb7a0ab 2024-11-23T05:42:58,509 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/805e1c6ea2ea499a97b0d8538d5b72ba to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/805e1c6ea2ea499a97b0d8538d5b72ba 2024-11-23T05:42:58,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/264c80e833cb46a8942b5026c3db4e1b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/264c80e833cb46a8942b5026c3db4e1b 2024-11-23T05:42:58,510 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d54528007c3649e1b59d9961fb33f614 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d54528007c3649e1b59d9961fb33f614 2024-11-23T05:42:58,511 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/66ec397f737941728639d28676188a66 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/66ec397f737941728639d28676188a66 2024-11-23T05:42:58,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/ee87403090484184a5018345a997696e to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/ee87403090484184a5018345a997696e 2024-11-23T05:42:58,512 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8e95f8daff5940c1aa2f0495133beca7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8e95f8daff5940c1aa2f0495133beca7 2024-11-23T05:42:58,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d89b7efb7ead441a8632728d097fece3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d89b7efb7ead441a8632728d097fece3 2024-11-23T05:42:58,513 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/5e8a39ac29944a2aacaadac3377a0f29 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/5e8a39ac29944a2aacaadac3377a0f29 2024-11-23T05:42:58,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/b85b63488771425a8418cd978fb063f3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/b85b63488771425a8418cd978fb063f3 2024-11-23T05:42:58,514 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3b1e0035440a491cb316de3afc1a39cc to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3b1e0035440a491cb316de3afc1a39cc 2024-11-23T05:42:58,515 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8dca9b0a0a5a482eaa79426502e8c42c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8dca9b0a0a5a482eaa79426502e8c42c 2024-11-23T05:42:58,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/97ce7dbb38834f98bc3123df0f9dd013 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/97ce7dbb38834f98bc3123df0f9dd013 2024-11-23T05:42:58,516 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3aba7490273b4a1fb774881fd56326ad to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/3aba7490273b4a1fb774881fd56326ad 2024-11-23T05:42:58,517 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/741a736fde314aff8379fad420a2e121 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/741a736fde314aff8379fad420a2e121 2024-11-23T05:42:58,518 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d6caa1bf0e874adca70da0d608521a54 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/d6caa1bf0e874adca70da0d608521a54 2024-11-23T05:42:58,519 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/01db9728e53846c3bd3ed8151980a9e3 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/01db9728e53846c3bd3ed8151980a9e3 2024-11-23T05:42:58,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] regionserver.HStore(2317): Moving the files [hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cc46b4b573b04c8d9a907d5bc06e25e1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/aef7ef4925484a0787ab6f1d4b9e5bf5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5c10d96ebe78426c875245f83bca34e5, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/722a6f005d9a437da209f6c5802b2bf1, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/9d6244c886004d0c97dc57311a690ff4, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/bd819b3fc42747d0917ef1cd74b73708, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cf667d8128a247419eca982eb6596b9f, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/3911e1ee4f574be6abe74972d6e9b81a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/1419234c028e48dcb187470ecb914c67, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/62c7c8050c3a409ab3ad1d82991469fd, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/8274dc580a2e48d0998e7e26163ce326, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/6ab69063359d4928aab611f4b76a1780, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/4bd28f2dfd674dc283f5fa196a04a2f2, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/642b339d2548496e8a436d83de2c5a05, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5d503136f51f435990464e1c8f60272a, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/b78fdfe3202748b088c63e7c15028726, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/52d6c015f1a840198c6764b5c818f592, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/26d572fcc14d430e8524559903ddcb80] to archive 2024-11-23T05:42:58,520 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(360): Archiving compacted files. 2024-11-23T05:42:58,521 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cc46b4b573b04c8d9a907d5bc06e25e1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cc46b4b573b04c8d9a907d5bc06e25e1 2024-11-23T05:42:58,522 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/aef7ef4925484a0787ab6f1d4b9e5bf5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/aef7ef4925484a0787ab6f1d4b9e5bf5 2024-11-23T05:42:58,523 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5c10d96ebe78426c875245f83bca34e5 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5c10d96ebe78426c875245f83bca34e5 2024-11-23T05:42:58,524 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/722a6f005d9a437da209f6c5802b2bf1 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/722a6f005d9a437da209f6c5802b2bf1 2024-11-23T05:42:58,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/9d6244c886004d0c97dc57311a690ff4 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/9d6244c886004d0c97dc57311a690ff4 2024-11-23T05:42:58,525 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/bd819b3fc42747d0917ef1cd74b73708 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/bd819b3fc42747d0917ef1cd74b73708 2024-11-23T05:42:58,526 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cf667d8128a247419eca982eb6596b9f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/cf667d8128a247419eca982eb6596b9f 2024-11-23T05:42:58,527 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/3911e1ee4f574be6abe74972d6e9b81a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/3911e1ee4f574be6abe74972d6e9b81a 2024-11-23T05:42:58,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/1419234c028e48dcb187470ecb914c67 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/1419234c028e48dcb187470ecb914c67 2024-11-23T05:42:58,528 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/62c7c8050c3a409ab3ad1d82991469fd to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/62c7c8050c3a409ab3ad1d82991469fd 2024-11-23T05:42:58,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/8274dc580a2e48d0998e7e26163ce326 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/8274dc580a2e48d0998e7e26163ce326 2024-11-23T05:42:58,529 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/6ab69063359d4928aab611f4b76a1780 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/6ab69063359d4928aab611f4b76a1780 2024-11-23T05:42:58,530 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/4bd28f2dfd674dc283f5fa196a04a2f2 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/4bd28f2dfd674dc283f5fa196a04a2f2 2024-11-23T05:42:58,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/642b339d2548496e8a436d83de2c5a05 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/642b339d2548496e8a436d83de2c5a05 2024-11-23T05:42:58,531 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5d503136f51f435990464e1c8f60272a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/5d503136f51f435990464e1c8f60272a 2024-11-23T05:42:58,532 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/b78fdfe3202748b088c63e7c15028726 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/b78fdfe3202748b088c63e7c15028726 2024-11-23T05:42:58,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/52d6c015f1a840198c6764b5c818f592 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/52d6c015f1a840198c6764b5c818f592 2024-11-23T05:42:58,533 DEBUG [StoreCloser-TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.-1 {}] backup.HFileArchiver(596): Archived from FileableStoreFile, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/26d572fcc14d430e8524559903ddcb80 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/26d572fcc14d430e8524559903ddcb80 2024-11-23T05:42:58,537 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/recovered.edits/303.seqid, newMaxSeqId=303, maxSeqId=4 2024-11-23T05:42:58,537 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1973): Closed TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. 2024-11-23T05:42:58,538 DEBUG [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] regionserver.HRegion(1676): Region close journal for f941d6128e412cb98f4108ef00511119: Waiting for close lock at 1732340576787Running coprocessor pre-close hooks at 1732340576787Disabling compacts and flushes for region at 1732340576787Disabling writes for close at 1732340576788 (+1 ms)Obtaining lock to block concurrent updates at 1732340576788Preparing flush snapshotting stores in f941d6128e412cb98f4108ef00511119 at 1732340576788Finished memstore snapshotting TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119., syncing WAL and waiting on mvcc, flushsize=dataSize=34350, getHeapSize=90720, getOffHeapSize=0, getCellsCount=750 at 1732340576789 (+1 ms)Flushing stores of TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119. at 1732340576790 (+1 ms)Flushing f941d6128e412cb98f4108ef00511119/A: creating writer at 1732340576790Flushing mob file f941d6128e412cb98f4108ef00511119/A: appending metadata at 1732340576798 (+8 ms)Flushing mob file f941d6128e412cb98f4108ef00511119/A: closing flushed file at 1732340576798Flushing f941d6128e412cb98f4108ef00511119/A: appending metadata at 1732340577210 (+412 ms)Flushing f941d6128e412cb98f4108ef00511119/A: closing flushed file at 1732340577211 (+1 ms)Flushing f941d6128e412cb98f4108ef00511119/B: creating writer at 1732340577627 (+416 ms)Flushing f941d6128e412cb98f4108ef00511119/B: appending metadata at 1732340577630 (+3 ms)Flushing f941d6128e412cb98f4108ef00511119/B: closing flushed file at 1732340577630Flushing f941d6128e412cb98f4108ef00511119/C: creating writer at 1732340578046 (+416 ms)Flushing f941d6128e412cb98f4108ef00511119/C: appending metadata at 1732340578050 (+4 ms)Flushing f941d6128e412cb98f4108ef00511119/C: closing flushed file at 1732340578050Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@1c8af855: reopening flushed file at 1732340578464 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@6a7dc10d: reopening flushed file at 1732340578469 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@11bccf77: reopening flushed file at 1732340578474 (+5 ms)Finished flush of dataSize ~33.54 KB/34350, heapSize ~88.59 KB/90720, currentSize=0 B/0 for f941d6128e412cb98f4108ef00511119 in 1691ms, sequenceid=300, compaction requested=true at 1732340578479 (+5 ms)Writing region close event to WAL at 1732340578535 (+56 ms)Running coprocessor post-close hooks at 1732340578537 (+2 ms)Closed at 1732340578537 2024-11-23T05:42:58,539 INFO [RS_CLOSE_REGION-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_REGION, pid=167}] handler.UnassignRegionHandler(157): Closed f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,540 INFO [PEWorker-5 {}] assignment.RegionStateStore(223): pid=166 updating hbase:meta row=f941d6128e412cb98f4108ef00511119, regionState=CLOSED 2024-11-23T05:42:58,541 DEBUG [MiniHBaseClusterRegionServer-EventLoopGroup-3-3 {}] procedure2.ProcedureFutureUtil(82): Going to wake up procedure pid=167, ppid=166, state=RUNNABLE, hasLock=false; CloseRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 because future has completed 2024-11-23T05:42:58,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=167, resume processing ppid=166 2024-11-23T05:42:58,543 INFO [PEWorker-3 {}] procedure2.ProcedureExecutor(1521): Finished pid=167, ppid=166, state=SUCCESS, hasLock=false; CloseRegionProcedure f941d6128e412cb98f4108ef00511119, server=9abfc7c2569e,45719,1732340394120 in 1.9080 sec 2024-11-23T05:42:58,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=166, resume processing ppid=165 2024-11-23T05:42:58,544 INFO [PEWorker-1 {}] procedure2.ProcedureExecutor(1521): Finished pid=166, ppid=165, state=SUCCESS, hasLock=false; TransitRegionStateProcedure table=TestAcidGuarantees, region=f941d6128e412cb98f4108ef00511119, UNASSIGN in 1.9130 sec 2024-11-23T05:42:58,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(2017): Finished subprocedure pid=165, resume processing ppid=164 2024-11-23T05:42:58,546 INFO [PEWorker-2 {}] procedure2.ProcedureExecutor(1521): Finished pid=165, ppid=164, state=SUCCESS, hasLock=false; CloseTableRegionsProcedure table=TestAcidGuarantees in 1.9170 sec 2024-11-23T05:42:58,548 DEBUG [PEWorker-4 {}] hbase.MetaTableAccessor(964): Put {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":2,"tag":[],"timestamp":"1732340578547"}]},"ts":"1732340578547"} 2024-11-23T05:42:58,549 INFO [PEWorker-4 {}] hbase.MetaTableAccessor(843): Updated tableName=TestAcidGuarantees, state=DISABLED in hbase:meta 2024-11-23T05:42:58,549 INFO [PEWorker-4 {}] procedure.DisableTableProcedure(296): Set TestAcidGuarantees to state=DISABLED 2024-11-23T05:42:58,551 INFO [PEWorker-4 {}] procedure2.ProcedureExecutor(1521): Finished pid=164, state=SUCCESS, hasLock=false; DisableTableProcedure table=TestAcidGuarantees in 1.9280 sec 2024-11-23T05:42:58,776 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=164 2024-11-23T05:42:58,777 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DISABLE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:58,778 INFO [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.HMaster$5(2570): Client=jenkins//172.17.0.2 delete TestAcidGuarantees 2024-11-23T05:42:58,780 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] procedure2.ProcedureExecutor(1139): Stored pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:58,781 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(103): Waiting for RIT for pid=168, state=RUNNABLE:DELETE_TABLE_PRE_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:58,782 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T05:42:58,783 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(115): Deleting regions from filesystem for pid=168, state=RUNNABLE:DELETE_TABLE_CLEAR_FS_LAYOUT, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:58,786 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,790 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C, FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/recovered.edits] 2024-11-23T05:42:58,795 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d3a70c4ef2944869cc7e2900400aa4b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/0d3a70c4ef2944869cc7e2900400aa4b 2024-11-23T05:42:58,797 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/387ac5ed0e3749958a1116ac8598f04b to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/387ac5ed0e3749958a1116ac8598f04b 2024-11-23T05:42:58,799 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/bdbcd9613cdc4a00b00c2a32295dbf7a to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/A/bdbcd9613cdc4a00b00c2a32295dbf7a 2024-11-23T05:42:58,803 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/78d4a77c22784d9fa4b73de78636fc07 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/78d4a77c22784d9fa4b73de78636fc07 2024-11-23T05:42:58,805 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8a0c53a162ab4a4c82b9c3cb3920881f to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8a0c53a162ab4a4c82b9c3cb3920881f 2024-11-23T05:42:58,807 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8a3b406546f74948bcc2461c7c0c0392 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/B/8a3b406546f74948bcc2461c7c0c0392 2024-11-23T05:42:58,811 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/7f888b1892c144778c49e1a09bde6271 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/7f888b1892c144778c49e1a09bde6271 2024-11-23T05:42:58,813 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/e193a15fe0f744338866cc5cf293e64c to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/e193a15fe0f744338866cc5cf293e64c 2024-11-23T05:42:58,815 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/ede7fbb3181a42cc88f6d1209e80e3c7 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/C/ede7fbb3181a42cc88f6d1209e80e3c7 2024-11-23T05:42:58,819 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/recovered.edits/303.seqid to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119/recovered.edits/303.seqid 2024-11-23T05:42:58,820 DEBUG [HFileArchiver-6 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/default/TestAcidGuarantees/f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,820 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(313): Archived TestAcidGuarantees regions 2024-11-23T05:42:58,820 DEBUG [PEWorker-5 {}] backup.HFileArchiver(131): ARCHIVING hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T05:42:58,822 DEBUG [PEWorker-5 {}] backup.HFileArchiver(159): Archiving [FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A] 2024-11-23T05:42:58,827 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230ff0447d3c204c8d884516df28841ce7_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411230ff0447d3c204c8d884516df28841ce7_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,829 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123185f72455fb9448ab8d9bd1d4eb0efed_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123185f72455fb9448ab8d9bd1d4eb0efed_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,831 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123262b9aac6085424b95020935ed16321d_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123262b9aac6085424b95020935ed16321d_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,833 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112331fe8caed1ab4c8483d87dfd9f00e349_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112331fe8caed1ab4c8483d87dfd9f00e349_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,835 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236aa190dc2a184c74a32e873040303b29_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236aa190dc2a184c74a32e873040303b29_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,837 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236e31977e02894ddaa60bb1c3ec11c457_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236e31977e02894ddaa60bb1c3ec11c457_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,838 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236ef39644121d444c9484a1e36c3eb697_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411236ef39644121d444c9484a1e36c3eb697_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,839 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237f66f89c6e67423e926ec905ce4b9942_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411237f66f89c6e67423e926ec905ce4b9942_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,841 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112382f1cdccdbf0401ebc012565d981dc73_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112382f1cdccdbf0401ebc012565d981dc73_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,842 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112388d41a2421ea4234b2d50ba0ed7ad95b_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112388d41a2421ea4234b2d50ba0ed7ad95b_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,843 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112399ddf95f65c342a2afd2acc9618eb866_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e2024112399ddf95f65c342a2afd2acc9618eb866_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,844 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239cb041d77fb6464abd3c167184c4d97f_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e202411239cb041d77fb6464abd3c167184c4d97f_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,845 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b5c6d74e72ee42ad8fb7c242c1a0f9c3_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123b5c6d74e72ee42ad8fb7c242c1a0f9c3_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,847 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123bc8b51562e0a4d50b867aeee1d2f76b4_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123bc8b51562e0a4d50b867aeee1d2f76b4_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,848 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d7bb7eb440a34df1b9c5286586a25078_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123d7bb7eb440a34df1b9c5286586a25078_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,849 DEBUG [PEWorker-5 {}] backup.HFileArchiver(596): Archived from FileablePath, hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e24f160a1c4d4a41b31b7516a3c8798e_f941d6128e412cb98f4108ef00511119 to hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/archive/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3/A/d41d8cd98f00b204e9800998ecf8427e20241123e24f160a1c4d4a41b31b7516a3c8798e_f941d6128e412cb98f4108ef00511119 2024-11-23T05:42:58,849 DEBUG [PEWorker-5 {}] backup.HFileArchiver(610): Deleted hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/mobdir/data/default/TestAcidGuarantees/6995762b1b921fda31b7b26a0c5785c3 2024-11-23T05:42:58,851 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(120): Deleting regions from META for pid=168, state=RUNNABLE:DELETE_TABLE_REMOVE_FROM_META, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:58,853 WARN [PEWorker-5 {}] procedure.DeleteTableProcedure(370): Deleting some vestigial 1 rows of TestAcidGuarantees from hbase:meta 2024-11-23T05:42:58,854 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(407): Removing 'TestAcidGuarantees' descriptor. 2024-11-23T05:42:58,855 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(126): Deleting assignment state for pid=168, state=RUNNABLE:DELETE_TABLE_UNASSIGN_REGIONS, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:58,855 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(397): Removing 'TestAcidGuarantees' from region states. 2024-11-23T05:42:58,856 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(723): Delete {"totalColumns":1,"row":"TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.","families":{"info":[{"qualifier":"","vlen":0,"tag":[],"timestamp":"1732340578855"}]},"ts":"9223372036854775807"} 2024-11-23T05:42:58,857 INFO [PEWorker-5 {}] assignment.RegionStateStore(562): Deleted 1 regions from META 2024-11-23T05:42:58,857 DEBUG [PEWorker-5 {}] assignment.RegionStateStore(563): Deleted regions: [{ENCODED => f941d6128e412cb98f4108ef00511119, NAME => 'TestAcidGuarantees,,1732340550353.f941d6128e412cb98f4108ef00511119.', STARTKEY => '', ENDKEY => ''}] 2024-11-23T05:42:58,857 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(401): Marking 'TestAcidGuarantees' as deleted. 2024-11-23T05:42:58,857 DEBUG [PEWorker-5 {}] hbase.MetaTableAccessor(964): Delete {"totalColumns":1,"row":"TestAcidGuarantees","families":{"table":[{"qualifier":"state","vlen":0,"tag":[],"timestamp":"1732340578857"}]},"ts":"9223372036854775807"} 2024-11-23T05:42:58,858 INFO [PEWorker-5 {}] hbase.MetaTableAccessor(867): Deleted table TestAcidGuarantees state from META 2024-11-23T05:42:58,859 DEBUG [PEWorker-5 {}] procedure.DeleteTableProcedure(133): Finished pid=168, state=RUNNABLE:DELETE_TABLE_POST_OPERATION, hasLock=true; DeleteTableProcedure table=TestAcidGuarantees 2024-11-23T05:42:58,860 INFO [PEWorker-5 {}] procedure2.ProcedureExecutor(1521): Finished pid=168, state=SUCCESS, hasLock=false; DeleteTableProcedure table=TestAcidGuarantees in 81 msec 2024-11-23T05:42:58,886 DEBUG [RpcServer.default.FPBQ.Fifo.handler=1,queue=0,port=40189 {}] master.MasterRpcServices(1377): Checking to see if procedure is done pid=168 2024-11-23T05:42:58,886 DEBUG [RPCClient-NioEventLoopGroup-4-4 {}] client.AsyncRegionLocator(219): Clear meta cache for TestAcidGuarantees 2024-11-23T05:42:58,886 INFO [RPCClient-NioEventLoopGroup-4-4 {}] client.RawAsyncHBaseAdmin$TableProcedureBiConsumer(2721): Operation: DELETE, Table Name: default:TestAcidGuarantees completed 2024-11-23T05:42:58,894 INFO [Time-limited test {}] hbase.ResourceChecker(175): after: TestAcidGuaranteesWithNoInMemCompaction#testMobGetAtomicity Thread=239 (was 241), OpenFileDescriptor=449 (was 451), MaxFileDescriptor=1048576 (was 1048576), SystemLoadAverage=369 (was 427), ProcessCount=11 (was 11), AvailableMemoryMB=8219 (was 8296) 2024-11-23T05:42:58,894 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1019): Shutting down minicluster 2024-11-23T05:42:58,895 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:42:58,895 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hbase.thirdparty.com.google.common.io.Closeables.close(Closeables.java:79) at org.apache.hadoop.hbase.HBaseTestingUtil.closeConnection(HBaseTestingUtil.java:2611) at org.apache.hadoop.hbase.HBaseTestingUtil.cleanup(HBaseTestingUtil.java:1065) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1034) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:42:58,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:58,895 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:58,895 INFO [Registry-endpoints-refresh-end-points {}] client.RegistryEndpointsRefresher(78): Registry end points refresher loop exited. 2024-11-23T05:42:58,895 DEBUG [Time-limited test {}] util.JVMClusterUtil(237): Shutting down HBase Cluster 2024-11-23T05:42:58,895 DEBUG [Time-limited test {}] util.JVMClusterUtil(257): Found active master hash=840105357, stopped=false 2024-11-23T05:42:58,895 INFO [Time-limited test {}] master.ServerManager(983): Cluster shutdown requested of master=9abfc7c2569e,40189,1732340393249 2024-11-23T05:42:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:42:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/running 2024-11-23T05:42:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:42:58,953 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:42:58,954 INFO [Time-limited test {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T05:42:58,955 INFO [Time-limited test {}] client.AsyncConnectionImpl(233): Connection has been closed by Time-limited test. 2024-11-23T05:42:58,955 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:42:58,956 DEBUG [Time-limited test {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.master.HMaster.lambda$shutdown$17(HMaster.java:3306) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.master.HMaster.shutdown(HMaster.java:3277) at org.apache.hadoop.hbase.util.JVMClusterUtil.shutdown(JVMClusterUtil.java:265) at org.apache.hadoop.hbase.LocalHBaseCluster.shutdown(LocalHBaseCluster.java:416) at org.apache.hadoop.hbase.SingleProcessHBaseCluster.shutdown(SingleProcessHBaseCluster.java:676) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniHBaseCluster(HBaseTestingUtil.java:1036) at org.apache.hadoop.hbase.HBaseTestingUtil.shutdownMiniCluster(HBaseTestingUtil.java:1020) at org.apache.hadoop.hbase.AcidGuaranteesTestBase.tearDownAfterClass(AcidGuaranteesTestBase.java:66) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:77) at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.base/java.lang.reflect.Method.invoke(Method.java:568) at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:59) at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12) at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:56) at org.junit.internal.runners.statements.RunAfters.invokeMethod(RunAfters.java:46) at org.junit.internal.runners.statements.RunAfters.evaluate(RunAfters.java:33) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:299) at org.junit.internal.runners.statements.FailOnTimeout$CallableStatement.call(FailOnTimeout.java:293) at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:42:58,956 DEBUG [zk-event-processor-pool-0 {}] zookeeper.ZKUtil(113): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Set watcher on znode that does not yet exist, /hbase/running 2024-11-23T05:42:58,956 DEBUG [Time-limited test {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:58,957 INFO [Time-limited test {}] regionserver.HRegionServer(2196): ***** STOPPING region server '9abfc7c2569e,45719,1732340394120' ***** 2024-11-23T05:42:58,957 INFO [Time-limited test {}] regionserver.HRegionServer(2210): STOPPED: Shutdown requested 2024-11-23T05:42:58,958 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HeapMemoryManager(220): Stopping 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] flush.RegionServerFlushTableProcedureManager(119): Stopping region server flush procedure manager gracefully. 2024-11-23T05:42:58,959 INFO [MemStoreFlusher.0 {}] regionserver.MemStoreFlusher$FlushHandler(374): MemStoreFlusher.0 exiting 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] snapshot.RegionServerSnapshotManager(137): Stopping RegionServerSnapshotManager gracefully. 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(959): stopping server 9abfc7c2569e,45719,1732340394120 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] client.AsyncConnectionImpl(233): Connection has been closed by RS:0;9abfc7c2569e:45719. 2024-11-23T05:42:58,959 DEBUG [RS:0;9abfc7c2569e:45719 {}] client.AsyncConnectionImpl(264): Call stack: at java.base/java.lang.Thread.getStackTrace(Thread.java:1619) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.lambda$close$5(AsyncConnectionImpl.java:235) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:187) at org.apache.hadoop.hbase.trace.TraceUtil.trace(TraceUtil.java:177) at org.apache.hadoop.hbase.client.AsyncConnectionImpl.close(AsyncConnectionImpl.java:229) at org.apache.hadoop.hbase.HBaseServerBase.closeClusterConnection(HBaseServerBase.java:457) at org.apache.hadoop.hbase.regionserver.HRegionServer.run(HRegionServer.java:962) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.runRegionServer(SingleProcessHBaseCluster.java:171) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer$1.run(SingleProcessHBaseCluster.java:155) at java.base/java.security.AccessController.doPrivileged(AccessController.java:399) at java.base/javax.security.auth.Subject.doAs(Subject.java:376) at org.apache.hadoop.security.UserGroupInformation.doAs(UserGroupInformation.java:1930) at org.apache.hadoop.hbase.security.User$SecureHadoopUser.runAs(User.java:322) at org.apache.hadoop.hbase.SingleProcessHBaseCluster$MiniHBaseClusterRegionServer.run(SingleProcessHBaseCluster.java:152) at java.base/java.lang.Thread.run(Thread.java:840) 2024-11-23T05:42:58,959 DEBUG [RS:0;9abfc7c2569e:45719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.CompactSplit(469): Waiting for Split Thread to finish... 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.CompactSplit(469): Waiting for Large Compaction Thread to finish... 2024-11-23T05:42:58,959 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.CompactSplit(469): Waiting for Small Compaction Thread to finish... 2024-11-23T05:42:58,960 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(3091): Received CLOSE for 1588230740 2024-11-23T05:42:58,960 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1321): Waiting on 1 regions to close 2024-11-23T05:42:58,961 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1325): Online Regions={1588230740=hbase:meta,,1.1588230740} 2024-11-23T05:42:58,961 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:42:58,961 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1722): Closing 1588230740, disabling compactions & flushes 2024-11-23T05:42:58,961 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1755): Closing region hbase:meta,,1.1588230740 2024-11-23T05:42:58,961 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1776): Time limited wait for close lock on hbase:meta,,1.1588230740 2024-11-23T05:42:58,961 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1843): Acquired close lock on hbase:meta,,1.1588230740 after waiting 0 ms 2024-11-23T05:42:58,961 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1853): Updates disabled for region hbase:meta,,1.1588230740 2024-11-23T05:42:58,961 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(2902): Flushing 1588230740 4/4 column families, dataSize=19.91 KB heapSize=35.20 KB 2024-11-23T05:42:58,981 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/info/6b3bebcf1cfd490b864a87e89d3cda08 is 95, key is TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570./info:/1732340432104/DeleteFamily/seqid=0 2024-11-23T05:42:58,984 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742419_1595 (size=6514) 2024-11-23T05:42:58,993 INFO [regionserver/9abfc7c2569e:0.leaseChecker {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:42:59,161 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:42:59,327 INFO [regionserver/9abfc7c2569e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: MemstoreFlusherChore was stopped 2024-11-23T05:42:59,328 INFO [regionserver/9abfc7c2569e:0.Chore.1 {}] hbase.ScheduledChore(149): Chore: CompactionChecker was stopped 2024-11-23T05:42:59,362 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:42:59,386 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=16.75 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/info/6b3bebcf1cfd490b864a87e89d3cda08 2024-11-23T05:42:59,411 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/ns/e210cfcad2a4421d882c60c7e82b78e3 is 93, key is TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570./ns:/1732340432065/DeleteFamily/seqid=0 2024-11-23T05:42:59,414 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742420_1596 (size=5906) 2024-11-23T05:42:59,562 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:42:59,762 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:42:59,816 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=608 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/ns/e210cfcad2a4421d882c60c7e82b78e3 2024-11-23T05:42:59,842 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/rep_barrier/5fb393c1c7d449e386947fee1bb63a47 is 102, key is TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570./rep_barrier:/1732340432065/DeleteFamily/seqid=0 2024-11-23T05:42:59,845 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742421_1597 (size=6025) 2024-11-23T05:42:59,962 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:43:00,163 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:43:00,247 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=588 B at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/rep_barrier/5fb393c1c7d449e386947fee1bb63a47 2024-11-23T05:43:00,272 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/table/34635ff6247d40fb816805c0103227a5 is 96, key is TestAcidGuarantees,,1732340398888.96c10c5615cf153b0faba5173b51e570./table:/1732340432065/DeleteFamily/seqid=0 2024-11-23T05:43:00,275 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742422_1598 (size=5936) 2024-11-23T05:43:00,363 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:43:00,564 DEBUG [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1351): Waiting on 1588230740 2024-11-23T05:43:00,677 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=1.99 KB at sequenceid=90 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/table/34635ff6247d40fb816805c0103227a5 2024-11-23T05:43:00,687 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/info/6b3bebcf1cfd490b864a87e89d3cda08 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/info/6b3bebcf1cfd490b864a87e89d3cda08 2024-11-23T05:43:00,690 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/info/6b3bebcf1cfd490b864a87e89d3cda08, entries=12, sequenceid=90, filesize=6.4 K 2024-11-23T05:43:00,691 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/ns/e210cfcad2a4421d882c60c7e82b78e3 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/ns/e210cfcad2a4421d882c60c7e82b78e3 2024-11-23T05:43:00,694 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/ns/e210cfcad2a4421d882c60c7e82b78e3, entries=8, sequenceid=90, filesize=5.8 K 2024-11-23T05:43:00,694 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/rep_barrier/5fb393c1c7d449e386947fee1bb63a47 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/rep_barrier/5fb393c1c7d449e386947fee1bb63a47 2024-11-23T05:43:00,697 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/rep_barrier/5fb393c1c7d449e386947fee1bb63a47, entries=6, sequenceid=90, filesize=5.9 K 2024-11-23T05:43:00,697 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/.tmp/table/34635ff6247d40fb816805c0103227a5 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/table/34635ff6247d40fb816805c0103227a5 2024-11-23T05:43:00,700 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/table/34635ff6247d40fb816805c0103227a5, entries=7, sequenceid=90, filesize=5.8 K 2024-11-23T05:43:00,701 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(3140): Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false 2024-11-23T05:43:00,705 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] wal.WALSplitUtil(410): Wrote file=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/data/hbase/meta/1588230740/recovered.edits/93.seqid, newMaxSeqId=93, maxSeqId=1 2024-11-23T05:43:00,705 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] coprocessor.CoprocessorHost(310): Stop coprocessor org.apache.hadoop.hbase.coprocessor.MultiRowMutationEndpoint 2024-11-23T05:43:00,705 INFO [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1973): Closed hbase:meta,,1.1588230740 2024-11-23T05:43:00,706 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] regionserver.HRegion(1676): Region close journal for 1588230740: Waiting for close lock at 1732340578960Running coprocessor pre-close hooks at 1732340578961 (+1 ms)Disabling compacts and flushes for region at 1732340578961Disabling writes for close at 1732340578961Obtaining lock to block concurrent updates at 1732340578961Preparing flush snapshotting stores in 1588230740 at 1732340578961Finished memstore snapshotting hbase:meta,,1.1588230740, syncing WAL and waiting on mvcc, flushsize=dataSize=20384, getHeapSize=35984, getOffHeapSize=0, getCellsCount=191 at 1732340578962 (+1 ms)Flushing stores of hbase:meta,,1.1588230740 at 1732340578962Flushing 1588230740/info: creating writer at 1732340578962Flushing 1588230740/info: appending metadata at 1732340578981 (+19 ms)Flushing 1588230740/info: closing flushed file at 1732340578981Flushing 1588230740/ns: creating writer at 1732340579396 (+415 ms)Flushing 1588230740/ns: appending metadata at 1732340579411 (+15 ms)Flushing 1588230740/ns: closing flushed file at 1732340579411Flushing 1588230740/rep_barrier: creating writer at 1732340579825 (+414 ms)Flushing 1588230740/rep_barrier: appending metadata at 1732340579842 (+17 ms)Flushing 1588230740/rep_barrier: closing flushed file at 1732340579842Flushing 1588230740/table: creating writer at 1732340580255 (+413 ms)Flushing 1588230740/table: appending metadata at 1732340580272 (+17 ms)Flushing 1588230740/table: closing flushed file at 1732340580272Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7667d010: reopening flushed file at 1732340580686 (+414 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@77720749: reopening flushed file at 1732340580690 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@74f12d1b: reopening flushed file at 1732340580694 (+4 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@60dbac34: reopening flushed file at 1732340580697 (+3 ms)Finished flush of dataSize ~19.91 KB/20384, heapSize ~35.14 KB/35984, currentSize=0 B/0 for 1588230740 in 1740ms, sequenceid=90, compaction requested=false at 1732340580701 (+4 ms)Writing region close event to WAL at 1732340580702 (+1 ms)Running coprocessor post-close hooks at 1732340580705 (+3 ms)Closed at 1732340580705 2024-11-23T05:43:00,706 DEBUG [RS_CLOSE_META-regionserver/9abfc7c2569e:0-0 {event_type=M_RS_CLOSE_META}] handler.CloseRegionHandler(117): Closed hbase:meta,,1.1588230740 2024-11-23T05:43:00,764 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(976): stopping server 9abfc7c2569e,45719,1732340394120; all regions closed. 2024-11-23T05:43:00,771 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741834_1010 (size=25344) 2024-11-23T05:43:00,775 DEBUG [RS:0;9abfc7c2569e:45719 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/oldWALs 2024-11-23T05:43:00,775 INFO [RS:0;9abfc7c2569e:45719 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C45719%2C1732340394120.meta:.meta(num 1732340398422) 2024-11-23T05:43:00,778 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741832_1008 (size=13356155) 2024-11-23T05:43:00,781 DEBUG [RS:0;9abfc7c2569e:45719 {}] wal.AbstractFSWAL(1256): Moved 1 WAL file(s) to /user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/oldWALs 2024-11-23T05:43:00,781 INFO [RS:0;9abfc7c2569e:45719 {}] wal.AbstractFSWAL(1259): Closed WAL: AsyncFSWAL 9abfc7c2569e%2C45719%2C1732340394120:(num 1732340397450) 2024-11-23T05:43:00,781 DEBUG [RS:0;9abfc7c2569e:45719 {}] ipc.AbstractRpcClient(514): Stopping rpc client 2024-11-23T05:43:00,781 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.LeaseManager(133): Closed leases 2024-11-23T05:43:00,781 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:43:00,782 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.ChoreService(370): Chore service for: regionserver/9abfc7c2569e:0 had [ScheduledChore name=ReplicationSourceStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=ReplicationSinkStatistics, period=300000, unit=MILLISECONDS, ScheduledChore name=CompactionThroughputTuner, period=60000, unit=MILLISECONDS] on shutdown 2024-11-23T05:43:00,782 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:43:00,782 INFO [regionserver/9abfc7c2569e:0.logRoller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:43:00,783 INFO [RS:0;9abfc7c2569e:45719 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:45719 2024-11-23T05:43:00,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/rs/9abfc7c2569e,45719,1732340394120 2024-11-23T05:43:00,811 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase/rs 2024-11-23T05:43:00,811 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:43:00,823 INFO [RegionServerTracker-0 {}] master.RegionServerTracker(171): RegionServer ephemeral node deleted, processing expiration [9abfc7c2569e,45719,1732340394120] 2024-11-23T05:43:00,831 DEBUG [RegionServerTracker-0 {}] zookeeper.RecoverableZooKeeper(212): Node /hbase/draining/9abfc7c2569e,45719,1732340394120 already deleted, retry=false 2024-11-23T05:43:00,832 INFO [RegionServerTracker-0 {}] master.ServerManager(688): Cluster shutdown set; 9abfc7c2569e,45719,1732340394120 expired; onlineServers=0 2024-11-23T05:43:00,832 INFO [RegionServerTracker-0 {}] master.HMaster(3321): ***** STOPPING master '9abfc7c2569e,40189,1732340393249' ***** 2024-11-23T05:43:00,832 INFO [RegionServerTracker-0 {}] master.HMaster(3323): STOPPED: Cluster shutdown set; onlineServer=0 2024-11-23T05:43:00,832 INFO [M:0;9abfc7c2569e:40189 {}] hbase.HBaseServerBase(455): Close async cluster connection 2024-11-23T05:43:00,832 INFO [M:0;9abfc7c2569e:40189 {}] hbase.HBaseServerBase(438): Shutdown chores and chore service 2024-11-23T05:43:00,832 DEBUG [M:0;9abfc7c2569e:40189 {}] cleaner.LogCleaner(198): Cancelling LogCleaner 2024-11-23T05:43:00,833 WARN [OldWALsCleaner-0 {}] cleaner.LogCleaner(186): Interrupted while cleaning old WALs, will try to clean it next round. Exiting. 2024-11-23T05:43:00,833 DEBUG [M:0;9abfc7c2569e:40189 {}] cleaner.HFileCleaner(335): Stopping file delete threads 2024-11-23T05:43:00,833 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732340397083 {}] cleaner.HFileCleaner(306): Exit Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.small.0-1732340397083,5,FailOnTimeoutGroup] 2024-11-23T05:43:00,833 DEBUG [master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732340397082 {}] cleaner.HFileCleaner(306): Exit Thread[master/9abfc7c2569e:0:becomeActiveMaster-HFileCleaner.large.0-1732340397082,5,FailOnTimeoutGroup] 2024-11-23T05:43:00,833 INFO [M:0;9abfc7c2569e:40189 {}] hbase.ChoreService(370): Chore service for: master/9abfc7c2569e:0 had [ScheduledChore name=FlushedSequenceIdFlusher, period=10800000, unit=MILLISECONDS] on shutdown 2024-11-23T05:43:00,833 INFO [M:0;9abfc7c2569e:40189 {}] hbase.HBaseServerBase(448): Shutdown executor service 2024-11-23T05:43:00,833 DEBUG [M:0;9abfc7c2569e:40189 {}] master.HMaster(1795): Stopping service threads 2024-11-23T05:43:00,834 INFO [M:0;9abfc7c2569e:40189 {}] procedure2.RemoteProcedureDispatcher(119): Stopping procedure remote dispatcher 2024-11-23T05:43:00,834 INFO [M:0;9abfc7c2569e:40189 {}] procedure2.ProcedureExecutor(723): Stopping 2024-11-23T05:43:00,834 ERROR [M:0;9abfc7c2569e:40189 {}] procedure2.ProcedureExecutor(763): There are still active thread in group java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10], see STDOUT java.lang.ThreadGroup[name=PEWorkerGroup,maxpri=10] Thread[HFileArchiver-6,5,PEWorkerGroup] 2024-11-23T05:43:00,835 INFO [M:0;9abfc7c2569e:40189 {}] region.RegionProcedureStore(113): Stopping the Region Procedure Store, isAbort=false 2024-11-23T05:43:00,835 DEBUG [normalizer-worker-0 {}] normalizer.RegionNormalizerWorker(193): interrupt detected. terminating. 2024-11-23T05:43:00,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeDeleted, state=SyncConnected, path=/hbase/master 2024-11-23T05:43:00,844 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=NodeChildrenChanged, state=SyncConnected, path=/hbase 2024-11-23T05:43:00,844 DEBUG [M:0;9abfc7c2569e:40189 {}] zookeeper.ZKUtil(347): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Unable to get data of znode /hbase/master because node does not exist (not an error) 2024-11-23T05:43:00,845 WARN [M:0;9abfc7c2569e:40189 {}] master.ActiveMasterManager(344): Failed get of master address: java.io.IOException: Can't get master address from ZooKeeper; znode data == null 2024-11-23T05:43:00,845 INFO [M:0;9abfc7c2569e:40189 {}] master.ServerManager(1139): Writing .lastflushedseqids file at: hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/.lastflushedseqids 2024-11-23T05:43:00,858 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742423_1599 (size=72) 2024-11-23T05:43:00,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:43:00,924 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): regionserver:45719-0x101677e349d0001, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:43:00,925 INFO [RS:0;9abfc7c2569e:45719 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:43:00,925 INFO [RS:0;9abfc7c2569e:45719 {}] regionserver.HRegionServer(1031): Exiting; stopping=9abfc7c2569e,45719,1732340394120; zookeeper connection closed. 2024-11-23T05:43:00,926 INFO [Shutdown of org.apache.hadoop.hbase.fs.HFileSystem@6b944d0f {}] hbase.SingleProcessHBaseCluster$SingleFileSystemShutdownThread(211): Hook closing fs=org.apache.hadoop.hbase.fs.HFileSystem@6b944d0f 2024-11-23T05:43:00,927 INFO [Time-limited test {}] util.JVMClusterUtil(335): Shutdown of 1 master(s) and 1 regionserver(s) complete 2024-11-23T05:43:01,258 INFO [M:0;9abfc7c2569e:40189 {}] assignment.AssignmentManager(395): Stopping assignment manager 2024-11-23T05:43:01,259 INFO [M:0;9abfc7c2569e:40189 {}] region.MasterRegion(195): Closing local region {ENCODED => 1595e783b53d99cd5eef43b6debb2682, NAME => 'master:store,,1.1595e783b53d99cd5eef43b6debb2682.', STARTKEY => '', ENDKEY => ''}, isAbort=false 2024-11-23T05:43:01,259 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(1722): Closing 1595e783b53d99cd5eef43b6debb2682, disabling compactions & flushes 2024-11-23T05:43:01,259 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(1755): Closing region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:43:01,259 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(1776): Time limited wait for close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:43:01,259 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(1843): Acquired close lock on master:store,,1.1595e783b53d99cd5eef43b6debb2682. after waiting 0 ms 2024-11-23T05:43:01,259 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(1853): Updates disabled for region master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:43:01,259 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(2902): Flushing 1595e783b53d99cd5eef43b6debb2682 4/4 column families, dataSize=772.38 KB heapSize=950.85 KB 2024-11-23T05:43:01,274 DEBUG [M:0;9abfc7c2569e:40189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc2a71716b784675885d2658fe3daf38 is 82, key is hbase:meta,,1/info:regioninfo/1732340398503/Put/seqid=0 2024-11-23T05:43:01,277 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742424_1600 (size=5672) 2024-11-23T05:43:01,678 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=504 B at sequenceid=2211 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc2a71716b784675885d2658fe3daf38 2024-11-23T05:43:01,701 DEBUG [M:0;9abfc7c2569e:40189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f366a4384da54ee19c84700040194e65 is 2276, key is \x00\x00\x00\x00\x00\x00\x00\x91/proc:d/1732340553452/Put/seqid=0 2024-11-23T05:43:01,704 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742425_1601 (size=43845) 2024-11-23T05:43:02,106 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=771.82 KB at sequenceid=2211 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f366a4384da54ee19c84700040194e65 2024-11-23T05:43:02,115 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f366a4384da54ee19c84700040194e65 2024-11-23T05:43:02,128 DEBUG [M:0;9abfc7c2569e:40189 {}] hfile.HFileWriterImpl(814): Len of the biggest cell in hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1afc725b3a0d4adb994a9fc54aacbd14 is 69, key is 9abfc7c2569e,45719,1732340394120/rs:state/1732340397206/Put/seqid=0 2024-11-23T05:43:02,131 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073742426_1602 (size=5156) 2024-11-23T05:43:02,533 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.DefaultStoreFlusher(81): Flushed memstore data size=65 B at sequenceid=2211 (bloomFilter=true), to=hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1afc725b3a0d4adb994a9fc54aacbd14 2024-11-23T05:43:02,542 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/info/cc2a71716b784675885d2658fe3daf38 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc2a71716b784675885d2658fe3daf38 2024-11-23T05:43:02,546 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/info/cc2a71716b784675885d2658fe3daf38, entries=8, sequenceid=2211, filesize=5.5 K 2024-11-23T05:43:02,546 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/proc/f366a4384da54ee19c84700040194e65 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f366a4384da54ee19c84700040194e65 2024-11-23T05:43:02,550 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.StoreFileReader(518): Loaded Delete Family Bloom (CompoundBloomFilter) metadata for f366a4384da54ee19c84700040194e65 2024-11-23T05:43:02,550 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/proc/f366a4384da54ee19c84700040194e65, entries=168, sequenceid=2211, filesize=42.8 K 2024-11-23T05:43:02,551 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegionFileSystem(442): Committing hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/.tmp/rs/1afc725b3a0d4adb994a9fc54aacbd14 as hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1afc725b3a0d4adb994a9fc54aacbd14 2024-11-23T05:43:02,554 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.HStore$StoreFlusherImpl(1990): Added hdfs://localhost:40229/user/jenkins/test-data/7182483e-c487-ee1d-e689-4beadb009f0d/MasterData/data/master/store/1595e783b53d99cd5eef43b6debb2682/rs/1afc725b3a0d4adb994a9fc54aacbd14, entries=1, sequenceid=2211, filesize=5.0 K 2024-11-23T05:43:02,555 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(3140): Finished flush of dataSize ~772.38 KB/790915, heapSize ~950.55 KB/973368, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2211, compaction requested=false 2024-11-23T05:43:02,556 INFO [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(1973): Closed master:store,,1.1595e783b53d99cd5eef43b6debb2682. 2024-11-23T05:43:02,557 DEBUG [M:0;9abfc7c2569e:40189 {}] regionserver.HRegion(1676): Region close journal for 1595e783b53d99cd5eef43b6debb2682: Waiting for close lock at 1732340581259Disabling compacts and flushes for region at 1732340581259Disabling writes for close at 1732340581259Obtaining lock to block concurrent updates at 1732340581259Preparing flush snapshotting stores in 1595e783b53d99cd5eef43b6debb2682 at 1732340581259Finished memstore snapshotting master:store,,1.1595e783b53d99cd5eef43b6debb2682., syncing WAL and waiting on mvcc, flushsize=dataSize=790915, getHeapSize=973608, getOffHeapSize=0, getCellsCount=2404 at 1732340581259Flushing stores of master:store,,1.1595e783b53d99cd5eef43b6debb2682. at 1732340581260 (+1 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: creating writer at 1732340581260Flushing 1595e783b53d99cd5eef43b6debb2682/info: appending metadata at 1732340581274 (+14 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/info: closing flushed file at 1732340581274Flushing 1595e783b53d99cd5eef43b6debb2682/proc: creating writer at 1732340581682 (+408 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: appending metadata at 1732340581700 (+18 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/proc: closing flushed file at 1732340581700Flushing 1595e783b53d99cd5eef43b6debb2682/rs: creating writer at 1732340582115 (+415 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: appending metadata at 1732340582128 (+13 ms)Flushing 1595e783b53d99cd5eef43b6debb2682/rs: closing flushed file at 1732340582128Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@5522e7a0: reopening flushed file at 1732340582541 (+413 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@4d8b2f6f: reopening flushed file at 1732340582546 (+5 ms)Flushing org.apache.hadoop.hbase.regionserver.HStore$StoreFlusherImpl@7eedbcdf: reopening flushed file at 1732340582550 (+4 ms)Finished flush of dataSize ~772.38 KB/790915, heapSize ~950.55 KB/973368, currentSize=0 B/0 for 1595e783b53d99cd5eef43b6debb2682 in 1296ms, sequenceid=2211, compaction requested=false at 1732340582555 (+5 ms)Writing region close event to WAL at 1732340582556 (+1 ms)Closed at 1732340582556 2024-11-23T05:43:02,559 INFO [Block report processor {}] blockmanagement.BlockManager(3777): BLOCK* addStoredBlock: 127.0.0.1:36825 is added to blk_1073741830_1006 (size=935821) 2024-11-23T05:43:02,560 INFO [master:store-WAL-Roller {}] wal.AbstractWALRoller(249): LogRoller exiting. 2024-11-23T05:43:02,560 INFO [M:0;9abfc7c2569e:40189 {}] flush.MasterFlushTableProcedureManager(90): stop: server shutting down. 2024-11-23T05:43:02,560 INFO [M:0;9abfc7c2569e:40189 {}] ipc.NettyRpcServer(345): Stopping server on /172.17.0.2:40189 2024-11-23T05:43:02,560 INFO [M:0;9abfc7c2569e:40189 {}] hbase.HBaseServerBase(479): Close zookeeper 2024-11-23T05:43:02,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:43:02,714 INFO [M:0;9abfc7c2569e:40189 {}] hbase.HBaseServerBase(486): Close table descriptors 2024-11-23T05:43:02,714 DEBUG [Time-limited test-EventThread {}] zookeeper.ZKWatcher(609): master:40189-0x101677e349d0000, quorum=127.0.0.1:53525, baseZNode=/hbase Received ZooKeeper Event, type=None, state=Closed, path=null 2024-11-23T05:43:02,724 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@4d1d0948{datanode,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/datanode} 2024-11-23T05:43:02,727 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@49b49063{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T05:43:02,727 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T05:43:02,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@58f8c8ee{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T05:43:02,727 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@6aa6ac55{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/hadoop.log.dir/,STOPPED} 2024-11-23T05:43:02,729 ERROR [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1416): Command processor encountered interrupt and exit. 2024-11-23T05:43:02,729 WARN [BP-789294947-172.17.0.2-1732340389511 heartbeating to localhost/127.0.0.1:40229 {}] datanode.IncrementalBlockReportManager(160): IncrementalBlockReportManager interrupted 2024-11-23T05:43:02,729 WARN [BP-789294947-172.17.0.2-1732340389511 heartbeating to localhost/127.0.0.1:40229 {}] datanode.BPServiceActor(925): Ending block pool service for: Block pool BP-789294947-172.17.0.2-1732340389511 (Datanode Uuid a2268abb-1a29-45ec-af1c-012ff91c8985) service to localhost/127.0.0.1:40229 2024-11-23T05:43:02,729 WARN [Command processor {}] datanode.BPServiceActor$CommandProcessingThread(1400): Ending command processor service for: Thread[Command processor,5,FailOnTimeoutGroup] 2024-11-23T05:43:02,731 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/data/data1/current/BP-789294947-172.17.0.2-1732340389511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T05:43:02,732 WARN [refreshUsed-/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/cluster_53447432-9d0f-10bd-092b-2f37b11121d9/data/data2/current/BP-789294947-172.17.0.2-1732340389511 {}] fs.CachingGetSpaceUsed$RefreshThread(231): Thread Interrupted waiting to refresh disk information: sleep interrupted 2024-11-23T05:43:02,732 WARN [Time-limited test {}] datanode.DataSetLockManager(261): not open lock leak check func 2024-11-23T05:43:02,738 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.w.WebAppContext@1bbf242f{hdfs,/,null,STOPPED}{jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/hdfs} 2024-11-23T05:43:02,739 INFO [Time-limited test {}] server.AbstractConnector(383): Stopped ServerConnector@34657887{HTTP/1.1, (http/1.1)}{localhost:0} 2024-11-23T05:43:02,739 INFO [Time-limited test {}] session.HouseKeeper(149): node0 Stopped scavenging 2024-11-23T05:43:02,739 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@41752735{static,/static,jar:file:/home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/local-repository/org/apache/hadoop/hadoop-hdfs/3.4.1/hadoop-hdfs-3.4.1-tests.jar!/webapps/static,STOPPED} 2024-11-23T05:43:02,739 INFO [Time-limited test {}] handler.ContextHandler(1159): Stopped o.e.j.s.ServletContextHandler@7def033e{logs,/logs,file:///home/jenkins/jenkins-home/workspace/HBase-Flaky-Tests_master/hbase-server/target/test-data/2131c87d-1613-3db6-d313-8c83f94667fb/hadoop.log.dir/,STOPPED} 2024-11-23T05:43:02,752 INFO [Time-limited test {}] zookeeper.MiniZooKeeperCluster(347): Shutdown MiniZK cluster with all ZK servers 2024-11-23T05:43:02,865 INFO [Time-limited test {}] hbase.HBaseTestingUtil(1026): Minicluster is down